verify.c revision f940128526dbe468a1951cce10c2fe5dbd23875f
1/* 2 * IO verification helpers 3 */ 4#include <unistd.h> 5#include <fcntl.h> 6#include <string.h> 7#include <assert.h> 8#include <pthread.h> 9#include <libgen.h> 10 11#include "fio.h" 12#include "verify.h" 13#include "trim.h" 14#include "lib/rand.h" 15#include "lib/hweight.h" 16 17#include "crc/md5.h" 18#include "crc/crc64.h" 19#include "crc/crc32.h" 20#include "crc/crc32c.h" 21#include "crc/crc16.h" 22#include "crc/crc7.h" 23#include "crc/sha256.h" 24#include "crc/sha512.h" 25#include "crc/sha1.h" 26 27static void populate_hdr(struct thread_data *td, struct io_u *io_u, 28 struct verify_header *hdr, unsigned int header_num, 29 unsigned int header_len); 30 31static void fill_pattern(struct thread_data *td, void *p, unsigned int len, 32 char *pattern, unsigned int pattern_bytes) 33{ 34 switch (pattern_bytes) { 35 case 0: 36 assert(0); 37 break; 38 case 1: 39 dprint(FD_VERIFY, "fill verify pattern b=0 len=%u\n", len); 40 memset(p, pattern[0], len); 41 break; 42 default: { 43 unsigned int i = 0, size = 0; 44 unsigned char *b = p; 45 46 dprint(FD_VERIFY, "fill verify pattern b=%d len=%u\n", 47 pattern_bytes, len); 48 49 while (i < len) { 50 size = pattern_bytes; 51 if (size > (len - i)) 52 size = len - i; 53 memcpy(b+i, pattern, size); 54 i += size; 55 } 56 break; 57 } 58 } 59} 60 61void fill_buffer_pattern(struct thread_data *td, void *p, unsigned int len) 62{ 63 fill_pattern(td, p, len, td->o.buffer_pattern, td->o.buffer_pattern_bytes); 64} 65 66void fill_verify_pattern(struct thread_data *td, void *p, unsigned int len, 67 struct io_u *io_u, unsigned long seed, int use_seed) 68{ 69 if (!td->o.verify_pattern_bytes) { 70 dprint(FD_VERIFY, "fill random bytes len=%u\n", len); 71 72 if (use_seed) 73 __fill_random_buf(p, len, seed); 74 else 75 io_u->rand_seed = fill_random_buf(&td->__verify_state, p, len); 76 return; 77 } 78 79 if (io_u->buf_filled_len >= len) { 80 dprint(FD_VERIFY, "using already filled verify pattern b=%d len=%u\n", 81 td->o.verify_pattern_bytes, len); 82 return; 83 } 84 85 fill_pattern(td, p, len, td->o.verify_pattern, td->o.verify_pattern_bytes); 86 87 io_u->buf_filled_len = len; 88} 89 90static unsigned int get_hdr_inc(struct thread_data *td, struct io_u *io_u) 91{ 92 unsigned int hdr_inc; 93 94 hdr_inc = io_u->buflen; 95 if (td->o.verify_interval && td->o.verify_interval <= io_u->buflen) 96 hdr_inc = td->o.verify_interval; 97 98 return hdr_inc; 99} 100 101static void fill_pattern_headers(struct thread_data *td, struct io_u *io_u, 102 unsigned long seed, int use_seed) 103{ 104 unsigned int hdr_inc, header_num; 105 struct verify_header *hdr; 106 void *p = io_u->buf; 107 108 fill_verify_pattern(td, p, io_u->buflen, io_u, seed, use_seed); 109 110 hdr_inc = get_hdr_inc(td, io_u); 111 header_num = 0; 112 for (; p < io_u->buf + io_u->buflen; p += hdr_inc) { 113 hdr = p; 114 populate_hdr(td, io_u, hdr, header_num, hdr_inc); 115 header_num++; 116 } 117} 118 119static void memswp(void *buf1, void *buf2, unsigned int len) 120{ 121 char swap[200]; 122 123 assert(len <= sizeof(swap)); 124 125 memcpy(&swap, buf1, len); 126 memcpy(buf1, buf2, len); 127 memcpy(buf2, &swap, len); 128} 129 130static void hexdump(void *buffer, int len) 131{ 132 unsigned char *p = buffer; 133 int i; 134 135 for (i = 0; i < len; i++) 136 log_err("%02x", p[i]); 137 log_err("\n"); 138} 139 140/* 141 * Prepare for separation of verify_header and checksum header 142 */ 143static inline unsigned int __hdr_size(int verify_type) 144{ 145 unsigned int len = 0; 146 147 switch (verify_type) { 148 case VERIFY_NONE: 149 case VERIFY_NULL: 150 len = 0; 151 break; 152 case VERIFY_MD5: 153 len = sizeof(struct vhdr_md5); 154 break; 155 case VERIFY_CRC64: 156 len = sizeof(struct vhdr_crc64); 157 break; 158 case VERIFY_CRC32C: 159 case VERIFY_CRC32: 160 case VERIFY_CRC32C_INTEL: 161 len = sizeof(struct vhdr_crc32); 162 break; 163 case VERIFY_CRC16: 164 len = sizeof(struct vhdr_crc16); 165 break; 166 case VERIFY_CRC7: 167 len = sizeof(struct vhdr_crc7); 168 break; 169 case VERIFY_SHA256: 170 len = sizeof(struct vhdr_sha256); 171 break; 172 case VERIFY_SHA512: 173 len = sizeof(struct vhdr_sha512); 174 break; 175 case VERIFY_META: 176 len = sizeof(struct vhdr_meta); 177 break; 178 case VERIFY_SHA1: 179 len = sizeof(struct vhdr_sha1); 180 break; 181 case VERIFY_PATTERN: 182 len = 0; 183 break; 184 default: 185 log_err("fio: unknown verify header!\n"); 186 assert(0); 187 } 188 189 return len + sizeof(struct verify_header); 190} 191 192static inline unsigned int hdr_size(struct verify_header *hdr) 193{ 194 return __hdr_size(hdr->verify_type); 195} 196 197static void *hdr_priv(struct verify_header *hdr) 198{ 199 void *priv = hdr; 200 201 return priv + sizeof(struct verify_header); 202} 203 204/* 205 * Verify container, pass info to verify handlers and allow them to 206 * pass info back in case of error 207 */ 208struct vcont { 209 /* 210 * Input 211 */ 212 struct io_u *io_u; 213 unsigned int hdr_num; 214 struct thread_data *td; 215 216 /* 217 * Output, only valid in case of error 218 */ 219 const char *name; 220 void *good_crc; 221 void *bad_crc; 222 unsigned int crc_len; 223}; 224 225static void dump_buf(char *buf, unsigned int len, unsigned long long offset, 226 const char *type, struct fio_file *f) 227{ 228 char *ptr, fname[256]; 229 int ret, fd; 230 231 ptr = strdup(f->file_name); 232 strcpy(fname, basename(ptr)); 233 234 sprintf(fname + strlen(fname), ".%llu.%s", offset, type); 235 236 fd = open(fname, O_CREAT | O_TRUNC | O_WRONLY, 0644); 237 if (fd < 0) { 238 perror("open verify buf file"); 239 return; 240 } 241 242 while (len) { 243 ret = write(fd, buf, len); 244 if (!ret) 245 break; 246 else if (ret < 0) { 247 perror("write verify buf file"); 248 break; 249 } 250 len -= ret; 251 buf += ret; 252 } 253 254 close(fd); 255 log_err(" %s data dumped as %s\n", type, fname); 256 free(ptr); 257} 258 259/* 260 * Dump the contents of the read block and re-generate the correct data 261 * and dump that too. 262 */ 263static void dump_verify_buffers(struct verify_header *hdr, struct vcont *vc) 264{ 265 struct thread_data *td = vc->td; 266 struct io_u *io_u = vc->io_u; 267 unsigned long hdr_offset; 268 struct io_u dummy; 269 void *buf; 270 271 if (!td->o.verify_dump) 272 return; 273 274 /* 275 * Dump the contents we just read off disk 276 */ 277 hdr_offset = vc->hdr_num * hdr->len; 278 279 dump_buf(io_u->buf + hdr_offset, hdr->len, io_u->offset + hdr_offset, 280 "received", vc->io_u->file); 281 282 /* 283 * Allocate a new buf and re-generate the original data 284 */ 285 buf = malloc(io_u->buflen); 286 dummy = *io_u; 287 dummy.buf = buf; 288 dummy.rand_seed = hdr->rand_seed; 289 dummy.buf_filled_len = 0; 290 dummy.buflen = io_u->buflen; 291 292 fill_pattern_headers(td, &dummy, hdr->rand_seed, 1); 293 294 dump_buf(buf + hdr_offset, hdr->len, io_u->offset + hdr_offset, 295 "expected", vc->io_u->file); 296 free(buf); 297} 298 299static void log_verify_failure(struct verify_header *hdr, struct vcont *vc) 300{ 301 unsigned long long offset; 302 303 offset = vc->io_u->offset; 304 offset += vc->hdr_num * hdr->len; 305 log_err("%.8s: verify failed at file %s offset %llu, length %u\n", 306 vc->name, vc->io_u->file->file_name, offset, hdr->len); 307 308 if (vc->good_crc && vc->bad_crc) { 309 log_err(" Expected CRC: "); 310 hexdump(vc->good_crc, vc->crc_len); 311 log_err(" Received CRC: "); 312 hexdump(vc->bad_crc, vc->crc_len); 313 } 314 315 dump_verify_buffers(hdr, vc); 316} 317 318/* 319 * Return data area 'header_num' 320 */ 321static inline void *io_u_verify_off(struct verify_header *hdr, struct vcont *vc) 322{ 323 return vc->io_u->buf + vc->hdr_num * hdr->len + hdr_size(hdr); 324} 325 326static int verify_io_u_pattern(struct verify_header *hdr, struct vcont *vc) 327{ 328 struct thread_data *td = vc->td; 329 struct io_u *io_u = vc->io_u; 330 char *buf, *pattern; 331 unsigned int header_size = __hdr_size(td->o.verify); 332 unsigned int len, mod, i, size, pattern_size; 333 334 pattern = td->o.verify_pattern; 335 pattern_size = td->o.verify_pattern_bytes; 336 if (pattern_size <= 1) 337 pattern_size = MAX_PATTERN_SIZE; 338 buf = (void *) hdr + header_size; 339 len = get_hdr_inc(td, io_u) - header_size; 340 mod = header_size % pattern_size; 341 342 for (i = 0; i < len; i += size) { 343 size = pattern_size - mod; 344 if (size > (len - i)) 345 size = len - i; 346 if (memcmp(buf + i, pattern + mod, size)) 347 /* Let the slow compare find the first mismatch byte. */ 348 break; 349 mod = 0; 350 } 351 352 for (; i < len; i++) { 353 if (buf[i] != pattern[mod]) { 354 unsigned int bits; 355 356 bits = hweight8(buf[i] ^ pattern[mod]); 357 log_err("fio: got pattern %x, wanted %x. Bad bits %d\n", 358 buf[i], pattern[mod], bits); 359 log_err("fio: bad pattern block offset %u\n", i); 360 dump_verify_buffers(hdr, vc); 361 return EILSEQ; 362 } 363 mod++; 364 if (mod == td->o.verify_pattern_bytes) 365 mod = 0; 366 } 367 368 return 0; 369} 370 371static int verify_io_u_meta(struct verify_header *hdr, struct vcont *vc) 372{ 373 struct thread_data *td = vc->td; 374 struct vhdr_meta *vh = hdr_priv(hdr); 375 struct io_u *io_u = vc->io_u; 376 int ret = EILSEQ; 377 378 dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len); 379 380 if (vh->offset == io_u->offset + vc->hdr_num * td->o.verify_interval) 381 ret = 0; 382 383 if (td->o.verify_pattern_bytes) 384 ret |= verify_io_u_pattern(hdr, vc); 385 386 /* 387 * For read-only workloads, the program cannot be certain of the 388 * last numberio written to a block. Checking of numberio will be done 389 * only for workloads that write data. 390 * For verify_only, numberio will be checked in the last iteration when 391 * the correct state of numberio, that would have been written to each 392 * block in a previous run of fio, has been reached. 393 */ 394 if (td_write(td) || td_rw(td)) 395 if (!td->o.verify_only || td->o.loops == 0) 396 if (vh->numberio != io_u->numberio) 397 ret = EILSEQ; 398 399 if (!ret) 400 return 0; 401 402 vc->name = "meta"; 403 log_verify_failure(hdr, vc); 404 return ret; 405} 406 407static int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc) 408{ 409 void *p = io_u_verify_off(hdr, vc); 410 struct vhdr_sha512 *vh = hdr_priv(hdr); 411 uint8_t sha512[128]; 412 struct fio_sha512_ctx sha512_ctx = { 413 .buf = sha512, 414 }; 415 416 dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", vc->io_u, hdr->len); 417 418 fio_sha512_init(&sha512_ctx); 419 fio_sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr)); 420 421 if (!memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512))) 422 return 0; 423 424 vc->name = "sha512"; 425 vc->good_crc = vh->sha512; 426 vc->bad_crc = sha512_ctx.buf; 427 vc->crc_len = sizeof(vh->sha512); 428 log_verify_failure(hdr, vc); 429 return EILSEQ; 430} 431 432static int verify_io_u_sha256(struct verify_header *hdr, struct vcont *vc) 433{ 434 void *p = io_u_verify_off(hdr, vc); 435 struct vhdr_sha256 *vh = hdr_priv(hdr); 436 uint8_t sha256[64]; 437 struct fio_sha256_ctx sha256_ctx = { 438 .buf = sha256, 439 }; 440 441 dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", vc->io_u, hdr->len); 442 443 fio_sha256_init(&sha256_ctx); 444 fio_sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr)); 445 446 if (!memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256))) 447 return 0; 448 449 vc->name = "sha256"; 450 vc->good_crc = vh->sha256; 451 vc->bad_crc = sha256_ctx.buf; 452 vc->crc_len = sizeof(vh->sha256); 453 log_verify_failure(hdr, vc); 454 return EILSEQ; 455} 456 457static int verify_io_u_sha1(struct verify_header *hdr, struct vcont *vc) 458{ 459 void *p = io_u_verify_off(hdr, vc); 460 struct vhdr_sha1 *vh = hdr_priv(hdr); 461 uint32_t sha1[5]; 462 struct fio_sha1_ctx sha1_ctx = { 463 .H = sha1, 464 }; 465 466 dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", vc->io_u, hdr->len); 467 468 fio_sha1_init(&sha1_ctx); 469 fio_sha1_update(&sha1_ctx, p, hdr->len - hdr_size(hdr)); 470 471 if (!memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1))) 472 return 0; 473 474 vc->name = "sha1"; 475 vc->good_crc = vh->sha1; 476 vc->bad_crc = sha1_ctx.H; 477 vc->crc_len = sizeof(vh->sha1); 478 log_verify_failure(hdr, vc); 479 return EILSEQ; 480} 481 482static int verify_io_u_crc7(struct verify_header *hdr, struct vcont *vc) 483{ 484 void *p = io_u_verify_off(hdr, vc); 485 struct vhdr_crc7 *vh = hdr_priv(hdr); 486 unsigned char c; 487 488 dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", vc->io_u, hdr->len); 489 490 c = fio_crc7(p, hdr->len - hdr_size(hdr)); 491 492 if (c == vh->crc7) 493 return 0; 494 495 vc->name = "crc7"; 496 vc->good_crc = &vh->crc7; 497 vc->bad_crc = &c; 498 vc->crc_len = 1; 499 log_verify_failure(hdr, vc); 500 return EILSEQ; 501} 502 503static int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc) 504{ 505 void *p = io_u_verify_off(hdr, vc); 506 struct vhdr_crc16 *vh = hdr_priv(hdr); 507 unsigned short c; 508 509 dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len); 510 511 c = fio_crc16(p, hdr->len - hdr_size(hdr)); 512 513 if (c == vh->crc16) 514 return 0; 515 516 vc->name = "crc16"; 517 vc->good_crc = &vh->crc16; 518 vc->bad_crc = &c; 519 vc->crc_len = 2; 520 log_verify_failure(hdr, vc); 521 return EILSEQ; 522} 523 524static int verify_io_u_crc64(struct verify_header *hdr, struct vcont *vc) 525{ 526 void *p = io_u_verify_off(hdr, vc); 527 struct vhdr_crc64 *vh = hdr_priv(hdr); 528 unsigned long long c; 529 530 dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", vc->io_u, hdr->len); 531 532 c = fio_crc64(p, hdr->len - hdr_size(hdr)); 533 534 if (c == vh->crc64) 535 return 0; 536 537 vc->name = "crc64"; 538 vc->good_crc = &vh->crc64; 539 vc->bad_crc = &c; 540 vc->crc_len = 8; 541 log_verify_failure(hdr, vc); 542 return EILSEQ; 543} 544 545static int verify_io_u_crc32(struct verify_header *hdr, struct vcont *vc) 546{ 547 void *p = io_u_verify_off(hdr, vc); 548 struct vhdr_crc32 *vh = hdr_priv(hdr); 549 uint32_t c; 550 551 dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", vc->io_u, hdr->len); 552 553 c = fio_crc32(p, hdr->len - hdr_size(hdr)); 554 555 if (c == vh->crc32) 556 return 0; 557 558 vc->name = "crc32"; 559 vc->good_crc = &vh->crc32; 560 vc->bad_crc = &c; 561 vc->crc_len = 4; 562 log_verify_failure(hdr, vc); 563 return EILSEQ; 564} 565 566static int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc) 567{ 568 void *p = io_u_verify_off(hdr, vc); 569 struct vhdr_crc32 *vh = hdr_priv(hdr); 570 uint32_t c; 571 572 dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len); 573 574 c = fio_crc32c(p, hdr->len - hdr_size(hdr)); 575 576 if (c == vh->crc32) 577 return 0; 578 579 vc->name = "crc32c"; 580 vc->good_crc = &vh->crc32; 581 vc->bad_crc = &c; 582 vc->crc_len = 4; 583 log_verify_failure(hdr, vc); 584 return EILSEQ; 585} 586 587static int verify_io_u_md5(struct verify_header *hdr, struct vcont *vc) 588{ 589 void *p = io_u_verify_off(hdr, vc); 590 struct vhdr_md5 *vh = hdr_priv(hdr); 591 uint32_t hash[MD5_HASH_WORDS]; 592 struct fio_md5_ctx md5_ctx = { 593 .hash = hash, 594 }; 595 596 dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", vc->io_u, hdr->len); 597 598 fio_md5_init(&md5_ctx); 599 fio_md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr)); 600 601 if (!memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash))) 602 return 0; 603 604 vc->name = "md5"; 605 vc->good_crc = vh->md5_digest; 606 vc->bad_crc = md5_ctx.hash; 607 vc->crc_len = sizeof(hash); 608 log_verify_failure(hdr, vc); 609 return EILSEQ; 610} 611 612/* 613 * Push IO verification to a separate thread 614 */ 615int verify_io_u_async(struct thread_data *td, struct io_u *io_u) 616{ 617 if (io_u->file) 618 put_file_log(td, io_u->file); 619 620 pthread_mutex_lock(&td->io_u_lock); 621 622 if (io_u->flags & IO_U_F_IN_CUR_DEPTH) { 623 td->cur_depth--; 624 io_u->flags &= ~IO_U_F_IN_CUR_DEPTH; 625 } 626 flist_add_tail(&io_u->verify_list, &td->verify_list); 627 io_u->flags |= IO_U_F_FREE_DEF; 628 pthread_mutex_unlock(&td->io_u_lock); 629 630 pthread_cond_signal(&td->verify_cond); 631 return 0; 632} 633 634static int verify_trimmed_io_u(struct thread_data *td, struct io_u *io_u) 635{ 636 static char zero_buf[1024]; 637 unsigned int this_len, len; 638 int ret = 0; 639 void *p; 640 641 if (!td->o.trim_zero) 642 return 0; 643 644 len = io_u->buflen; 645 p = io_u->buf; 646 do { 647 this_len = sizeof(zero_buf); 648 if (this_len > len) 649 this_len = len; 650 if (memcmp(p, zero_buf, this_len)) { 651 ret = EILSEQ; 652 break; 653 } 654 len -= this_len; 655 p += this_len; 656 } while (len); 657 658 if (!ret) 659 return 0; 660 661 log_err("trim: verify failed at file %s offset %llu, length %lu" 662 ", block offset %lu\n", 663 io_u->file->file_name, io_u->offset, io_u->buflen, 664 (unsigned long) (p - io_u->buf)); 665 return ret; 666} 667 668static int verify_header(struct io_u *io_u, struct verify_header *hdr) 669{ 670 void *p = hdr; 671 uint32_t crc; 672 673 if (hdr->magic != FIO_HDR_MAGIC) 674 return 1; 675 if (hdr->len > io_u->buflen) 676 return 2; 677 if (hdr->rand_seed != io_u->rand_seed) 678 return 3; 679 680 crc = fio_crc32c(p, offsetof(struct verify_header, crc32)); 681 if (crc == hdr->crc32) 682 return 0; 683 log_err("fio: verify header crc %x, calculated %x\n", hdr->crc32, crc); 684 return 4; 685} 686 687int verify_io_u(struct thread_data *td, struct io_u *io_u) 688{ 689 struct verify_header *hdr; 690 unsigned int header_size, hdr_inc, hdr_num = 0; 691 void *p; 692 int ret; 693 694 if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ) 695 return 0; 696 if (io_u->flags & IO_U_F_TRIMMED) { 697 ret = verify_trimmed_io_u(td, io_u); 698 goto done; 699 } 700 701 hdr_inc = get_hdr_inc(td, io_u); 702 703 ret = 0; 704 for (p = io_u->buf; p < io_u->buf + io_u->buflen; 705 p += hdr_inc, hdr_num++) { 706 struct vcont vc = { 707 .io_u = io_u, 708 .hdr_num = hdr_num, 709 .td = td, 710 }; 711 unsigned int verify_type; 712 713 if (ret && td->o.verify_fatal) 714 break; 715 716 header_size = __hdr_size(td->o.verify); 717 if (td->o.verify_offset) 718 memswp(p, p + td->o.verify_offset, header_size); 719 hdr = p; 720 721 /* 722 * Make rand_seed check pass when have verifysort or 723 * verify_backlog. 724 */ 725 if (td->o.verifysort || (td->flags & TD_F_VER_BACKLOG)) 726 io_u->rand_seed = hdr->rand_seed; 727 728 ret = verify_header(io_u, hdr); 729 switch (ret) { 730 case 0: 731 break; 732 case 1: 733 log_err("verify: bad magic header %x, wanted %x at " 734 "file %s offset %llu, length %u\n", 735 hdr->magic, FIO_HDR_MAGIC, 736 io_u->file->file_name, 737 io_u->offset + hdr_num * hdr->len, hdr->len); 738 return EILSEQ; 739 break; 740 case 2: 741 log_err("fio: verify header exceeds buffer length (%u " 742 "> %lu)\n", hdr->len, io_u->buflen); 743 return EILSEQ; 744 break; 745 case 3: 746 log_err("verify: bad header rand_seed %"PRIu64 747 ", wanted %"PRIu64" at file %s offset %llu, " 748 "length %u\n", 749 hdr->rand_seed, io_u->rand_seed, 750 io_u->file->file_name, 751 io_u->offset + hdr_num * hdr->len, hdr->len); 752 return EILSEQ; 753 break; 754 case 4: 755 return EILSEQ; 756 break; 757 default: 758 log_err("verify: unknown header error at file %s " 759 "offset %llu, length %u\n", 760 io_u->file->file_name, 761 io_u->offset + hdr_num * hdr->len, hdr->len); 762 return EILSEQ; 763 } 764 765 if (td->o.verify != VERIFY_NONE) 766 verify_type = td->o.verify; 767 else 768 verify_type = hdr->verify_type; 769 770 switch (verify_type) { 771 case VERIFY_MD5: 772 ret = verify_io_u_md5(hdr, &vc); 773 break; 774 case VERIFY_CRC64: 775 ret = verify_io_u_crc64(hdr, &vc); 776 break; 777 case VERIFY_CRC32C: 778 case VERIFY_CRC32C_INTEL: 779 ret = verify_io_u_crc32c(hdr, &vc); 780 break; 781 case VERIFY_CRC32: 782 ret = verify_io_u_crc32(hdr, &vc); 783 break; 784 case VERIFY_CRC16: 785 ret = verify_io_u_crc16(hdr, &vc); 786 break; 787 case VERIFY_CRC7: 788 ret = verify_io_u_crc7(hdr, &vc); 789 break; 790 case VERIFY_SHA256: 791 ret = verify_io_u_sha256(hdr, &vc); 792 break; 793 case VERIFY_SHA512: 794 ret = verify_io_u_sha512(hdr, &vc); 795 break; 796 case VERIFY_META: 797 ret = verify_io_u_meta(hdr, &vc); 798 break; 799 case VERIFY_SHA1: 800 ret = verify_io_u_sha1(hdr, &vc); 801 break; 802 case VERIFY_PATTERN: 803 ret = verify_io_u_pattern(hdr, &vc); 804 break; 805 default: 806 log_err("Bad verify type %u\n", hdr->verify_type); 807 ret = EINVAL; 808 } 809 810 if (ret && verify_type != hdr->verify_type) 811 log_err("fio: verify type mismatch (%u media, %u given)\n", 812 hdr->verify_type, verify_type); 813 } 814 815done: 816 if (ret && td->o.verify_fatal) 817 td->terminate = 1; 818 819 return ret; 820} 821 822static void fill_meta(struct verify_header *hdr, struct thread_data *td, 823 struct io_u *io_u, unsigned int header_num) 824{ 825 struct vhdr_meta *vh = hdr_priv(hdr); 826 827 vh->thread = td->thread_number; 828 829 vh->time_sec = io_u->start_time.tv_sec; 830 vh->time_usec = io_u->start_time.tv_usec; 831 832 vh->numberio = io_u->numberio; 833 834 vh->offset = io_u->offset + header_num * td->o.verify_interval; 835} 836 837static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len) 838{ 839 struct vhdr_sha512 *vh = hdr_priv(hdr); 840 struct fio_sha512_ctx sha512_ctx = { 841 .buf = vh->sha512, 842 }; 843 844 fio_sha512_init(&sha512_ctx); 845 fio_sha512_update(&sha512_ctx, p, len); 846} 847 848static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len) 849{ 850 struct vhdr_sha256 *vh = hdr_priv(hdr); 851 struct fio_sha256_ctx sha256_ctx = { 852 .buf = vh->sha256, 853 }; 854 855 fio_sha256_init(&sha256_ctx); 856 fio_sha256_update(&sha256_ctx, p, len); 857} 858 859static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len) 860{ 861 struct vhdr_sha1 *vh = hdr_priv(hdr); 862 struct fio_sha1_ctx sha1_ctx = { 863 .H = vh->sha1, 864 }; 865 866 fio_sha1_init(&sha1_ctx); 867 fio_sha1_update(&sha1_ctx, p, len); 868} 869 870static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len) 871{ 872 struct vhdr_crc7 *vh = hdr_priv(hdr); 873 874 vh->crc7 = fio_crc7(p, len); 875} 876 877static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len) 878{ 879 struct vhdr_crc16 *vh = hdr_priv(hdr); 880 881 vh->crc16 = fio_crc16(p, len); 882} 883 884static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len) 885{ 886 struct vhdr_crc32 *vh = hdr_priv(hdr); 887 888 vh->crc32 = fio_crc32(p, len); 889} 890 891static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len) 892{ 893 struct vhdr_crc32 *vh = hdr_priv(hdr); 894 895 vh->crc32 = fio_crc32c(p, len); 896} 897 898static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len) 899{ 900 struct vhdr_crc64 *vh = hdr_priv(hdr); 901 902 vh->crc64 = fio_crc64(p, len); 903} 904 905static void fill_md5(struct verify_header *hdr, void *p, unsigned int len) 906{ 907 struct vhdr_md5 *vh = hdr_priv(hdr); 908 struct fio_md5_ctx md5_ctx = { 909 .hash = (uint32_t *) vh->md5_digest, 910 }; 911 912 fio_md5_init(&md5_ctx); 913 fio_md5_update(&md5_ctx, p, len); 914} 915 916static void populate_hdr(struct thread_data *td, struct io_u *io_u, 917 struct verify_header *hdr, unsigned int header_num, 918 unsigned int header_len) 919{ 920 unsigned int data_len; 921 void *data, *p; 922 923 p = (void *) hdr; 924 925 hdr->magic = FIO_HDR_MAGIC; 926 hdr->verify_type = td->o.verify; 927 hdr->len = header_len; 928 hdr->rand_seed = io_u->rand_seed; 929 hdr->crc32 = fio_crc32c(p, offsetof(struct verify_header, crc32)); 930 931 data_len = header_len - hdr_size(hdr); 932 933 data = p + hdr_size(hdr); 934 switch (td->o.verify) { 935 case VERIFY_MD5: 936 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n", 937 io_u, hdr->len); 938 fill_md5(hdr, data, data_len); 939 break; 940 case VERIFY_CRC64: 941 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n", 942 io_u, hdr->len); 943 fill_crc64(hdr, data, data_len); 944 break; 945 case VERIFY_CRC32C: 946 case VERIFY_CRC32C_INTEL: 947 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n", 948 io_u, hdr->len); 949 fill_crc32c(hdr, data, data_len); 950 break; 951 case VERIFY_CRC32: 952 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n", 953 io_u, hdr->len); 954 fill_crc32(hdr, data, data_len); 955 break; 956 case VERIFY_CRC16: 957 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n", 958 io_u, hdr->len); 959 fill_crc16(hdr, data, data_len); 960 break; 961 case VERIFY_CRC7: 962 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n", 963 io_u, hdr->len); 964 fill_crc7(hdr, data, data_len); 965 break; 966 case VERIFY_SHA256: 967 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n", 968 io_u, hdr->len); 969 fill_sha256(hdr, data, data_len); 970 break; 971 case VERIFY_SHA512: 972 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n", 973 io_u, hdr->len); 974 fill_sha512(hdr, data, data_len); 975 break; 976 case VERIFY_META: 977 dprint(FD_VERIFY, "fill meta io_u %p, len %u\n", 978 io_u, hdr->len); 979 fill_meta(hdr, td, io_u, header_num); 980 break; 981 case VERIFY_SHA1: 982 dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n", 983 io_u, hdr->len); 984 fill_sha1(hdr, data, data_len); 985 break; 986 case VERIFY_PATTERN: 987 /* nothing to do here */ 988 break; 989 default: 990 log_err("fio: bad verify type: %d\n", td->o.verify); 991 assert(0); 992 } 993 if (td->o.verify_offset) 994 memswp(p, p + td->o.verify_offset, hdr_size(hdr)); 995} 996 997/* 998 * fill body of io_u->buf with random data and add a header with the 999 * checksum of choice 1000 */ 1001void populate_verify_io_u(struct thread_data *td, struct io_u *io_u) 1002{ 1003 if (td->o.verify == VERIFY_NULL) 1004 return; 1005 1006 io_u->numberio = td->io_issues[io_u->ddir]; 1007 1008 fill_pattern_headers(td, io_u, 0, 0); 1009} 1010 1011int get_next_verify(struct thread_data *td, struct io_u *io_u) 1012{ 1013 struct io_piece *ipo = NULL; 1014 1015 /* 1016 * this io_u is from a requeue, we already filled the offsets 1017 */ 1018 if (io_u->file) 1019 return 0; 1020 1021 if (!RB_EMPTY_ROOT(&td->io_hist_tree)) { 1022 struct rb_node *n = rb_first(&td->io_hist_tree); 1023 1024 ipo = rb_entry(n, struct io_piece, rb_node); 1025 1026 /* 1027 * Ensure that the associated IO has completed 1028 */ 1029 read_barrier(); 1030 if (ipo->flags & IP_F_IN_FLIGHT) 1031 goto nothing; 1032 1033 rb_erase(n, &td->io_hist_tree); 1034 assert(ipo->flags & IP_F_ONRB); 1035 ipo->flags &= ~IP_F_ONRB; 1036 } else if (!flist_empty(&td->io_hist_list)) { 1037 ipo = flist_entry(td->io_hist_list.next, struct io_piece, list); 1038 1039 /* 1040 * Ensure that the associated IO has completed 1041 */ 1042 read_barrier(); 1043 if (ipo->flags & IP_F_IN_FLIGHT) 1044 goto nothing; 1045 1046 flist_del(&ipo->list); 1047 assert(ipo->flags & IP_F_ONLIST); 1048 ipo->flags &= ~IP_F_ONLIST; 1049 } 1050 1051 if (ipo) { 1052 td->io_hist_len--; 1053 1054 io_u->offset = ipo->offset; 1055 io_u->buflen = ipo->len; 1056 io_u->numberio = ipo->numberio; 1057 io_u->file = ipo->file; 1058 io_u->flags |= IO_U_F_VER_LIST; 1059 1060 if (ipo->flags & IP_F_TRIMMED) 1061 io_u->flags |= IO_U_F_TRIMMED; 1062 1063 if (!fio_file_open(io_u->file)) { 1064 int r = td_io_open_file(td, io_u->file); 1065 1066 if (r) { 1067 dprint(FD_VERIFY, "failed file %s open\n", 1068 io_u->file->file_name); 1069 return 1; 1070 } 1071 } 1072 1073 get_file(ipo->file); 1074 assert(fio_file_open(io_u->file)); 1075 io_u->ddir = DDIR_READ; 1076 io_u->xfer_buf = io_u->buf; 1077 io_u->xfer_buflen = io_u->buflen; 1078 1079 remove_trim_entry(td, ipo); 1080 free(ipo); 1081 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u); 1082 1083 if (!td->o.verify_pattern_bytes) { 1084 io_u->rand_seed = __rand(&td->__verify_state); 1085 if (sizeof(int) != sizeof(long *)) 1086 io_u->rand_seed *= __rand(&td->__verify_state); 1087 } 1088 return 0; 1089 } 1090 1091nothing: 1092 dprint(FD_VERIFY, "get_next_verify: empty\n"); 1093 return 1; 1094} 1095 1096void fio_verify_init(struct thread_data *td) 1097{ 1098 if (td->o.verify == VERIFY_CRC32C_INTEL || 1099 td->o.verify == VERIFY_CRC32C) { 1100 crc32c_intel_probe(); 1101 } 1102} 1103 1104static void *verify_async_thread(void *data) 1105{ 1106 struct thread_data *td = data; 1107 struct io_u *io_u; 1108 int ret = 0; 1109 1110 if (td->o.verify_cpumask_set && 1111 fio_setaffinity(td->pid, td->o.verify_cpumask)) { 1112 log_err("fio: failed setting verify thread affinity\n"); 1113 goto done; 1114 } 1115 1116 do { 1117 FLIST_HEAD(list); 1118 1119 read_barrier(); 1120 if (td->verify_thread_exit) 1121 break; 1122 1123 pthread_mutex_lock(&td->io_u_lock); 1124 1125 while (flist_empty(&td->verify_list) && 1126 !td->verify_thread_exit) { 1127 ret = pthread_cond_wait(&td->verify_cond, 1128 &td->io_u_lock); 1129 if (ret) { 1130 pthread_mutex_unlock(&td->io_u_lock); 1131 break; 1132 } 1133 } 1134 1135 flist_splice_init(&td->verify_list, &list); 1136 pthread_mutex_unlock(&td->io_u_lock); 1137 1138 if (flist_empty(&list)) 1139 continue; 1140 1141 while (!flist_empty(&list)) { 1142 io_u = flist_entry(list.next, struct io_u, verify_list); 1143 flist_del(&io_u->verify_list); 1144 1145 ret = verify_io_u(td, io_u); 1146 put_io_u(td, io_u); 1147 if (!ret) 1148 continue; 1149 if (td_non_fatal_error(td, ERROR_TYPE_VERIFY_BIT, ret)) { 1150 update_error_count(td, ret); 1151 td_clear_error(td); 1152 ret = 0; 1153 } 1154 } 1155 } while (!ret); 1156 1157 if (ret) { 1158 td_verror(td, ret, "async_verify"); 1159 if (td->o.verify_fatal) 1160 td->terminate = 1; 1161 } 1162 1163done: 1164 pthread_mutex_lock(&td->io_u_lock); 1165 td->nr_verify_threads--; 1166 pthread_mutex_unlock(&td->io_u_lock); 1167 1168 pthread_cond_signal(&td->free_cond); 1169 return NULL; 1170} 1171 1172int verify_async_init(struct thread_data *td) 1173{ 1174 int i, ret; 1175 pthread_attr_t attr; 1176 1177 pthread_attr_init(&attr); 1178 pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN); 1179 1180 td->verify_thread_exit = 0; 1181 1182 td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async); 1183 for (i = 0; i < td->o.verify_async; i++) { 1184 ret = pthread_create(&td->verify_threads[i], &attr, 1185 verify_async_thread, td); 1186 if (ret) { 1187 log_err("fio: async verify creation failed: %s\n", 1188 strerror(ret)); 1189 break; 1190 } 1191 ret = pthread_detach(td->verify_threads[i]); 1192 if (ret) { 1193 log_err("fio: async verify thread detach failed: %s\n", 1194 strerror(ret)); 1195 break; 1196 } 1197 td->nr_verify_threads++; 1198 } 1199 1200 pthread_attr_destroy(&attr); 1201 1202 if (i != td->o.verify_async) { 1203 log_err("fio: only %d verify threads started, exiting\n", i); 1204 td->verify_thread_exit = 1; 1205 write_barrier(); 1206 pthread_cond_broadcast(&td->verify_cond); 1207 return 1; 1208 } 1209 1210 return 0; 1211} 1212 1213void verify_async_exit(struct thread_data *td) 1214{ 1215 td->verify_thread_exit = 1; 1216 write_barrier(); 1217 pthread_cond_broadcast(&td->verify_cond); 1218 1219 pthread_mutex_lock(&td->io_u_lock); 1220 1221 while (td->nr_verify_threads) 1222 pthread_cond_wait(&td->free_cond, &td->io_u_lock); 1223 1224 pthread_mutex_unlock(&td->io_u_lock); 1225 free(td->verify_threads); 1226 td->verify_threads = NULL; 1227} 1228