verify.c revision e4ad68b1364aebfe949e2d6a649c7b1001ebca07
1/* 2 * IO verification helpers 3 */ 4#include <unistd.h> 5#include <fcntl.h> 6#include <string.h> 7#include <assert.h> 8#include <pthread.h> 9#include <libgen.h> 10 11#include "fio.h" 12#include "verify.h" 13#include "smalloc.h" 14#include "trim.h" 15#include "lib/rand.h" 16 17#include "crc/md5.h" 18#include "crc/crc64.h" 19#include "crc/crc32.h" 20#include "crc/crc32c.h" 21#include "crc/crc16.h" 22#include "crc/crc7.h" 23#include "crc/sha256.h" 24#include "crc/sha512.h" 25#include "crc/sha1.h" 26 27static void populate_hdr(struct thread_data *td, struct io_u *io_u, 28 struct verify_header *hdr, unsigned int header_num, 29 unsigned int header_len); 30 31void fill_pattern(struct thread_data *td, void *p, unsigned int len, struct io_u *io_u, unsigned long seed, int use_seed) 32{ 33 switch (td->o.verify_pattern_bytes) { 34 case 0: 35 dprint(FD_VERIFY, "fill random bytes len=%u\n", len); 36 if (use_seed) 37 __fill_random_buf(p, len, seed); 38 else 39 io_u->rand_seed = fill_random_buf(&td->buf_state, p, len); 40 break; 41 case 1: 42 if (io_u->buf_filled_len >= len) { 43 dprint(FD_VERIFY, "using already filled verify pattern b=0 len=%u\n", len); 44 return; 45 } 46 dprint(FD_VERIFY, "fill verify pattern b=0 len=%u\n", len); 47 memset(p, td->o.verify_pattern[0], len); 48 io_u->buf_filled_len = len; 49 break; 50 default: { 51 unsigned int i = 0, size = 0; 52 unsigned char *b = p; 53 54 if (io_u->buf_filled_len >= len) { 55 dprint(FD_VERIFY, "using already filled verify pattern b=%d len=%u\n", 56 td->o.verify_pattern_bytes, len); 57 return; 58 } 59 60 dprint(FD_VERIFY, "fill verify pattern b=%d len=%u\n", 61 td->o.verify_pattern_bytes, len); 62 63 while (i < len) { 64 size = td->o.verify_pattern_bytes; 65 if (size > (len - i)) 66 size = len - i; 67 memcpy(b+i, td->o.verify_pattern, size); 68 i += size; 69 } 70 io_u->buf_filled_len = len; 71 break; 72 } 73 } 74} 75 76static unsigned int get_hdr_inc(struct thread_data *td, struct io_u *io_u) 77{ 78 unsigned int hdr_inc; 79 80 hdr_inc = io_u->buflen; 81 if (td->o.verify_interval) 82 hdr_inc = td->o.verify_interval; 83 84 return hdr_inc; 85} 86 87static void fill_pattern_headers(struct thread_data *td, struct io_u *io_u, 88 unsigned long seed, int use_seed) 89{ 90 unsigned int hdr_inc, header_num; 91 struct verify_header *hdr; 92 void *p = io_u->buf; 93 94 fill_pattern(td, p, io_u->buflen, io_u, seed, use_seed); 95 96 hdr_inc = get_hdr_inc(td, io_u); 97 header_num = 0; 98 for (; p < io_u->buf + io_u->buflen; p += hdr_inc) { 99 hdr = p; 100 populate_hdr(td, io_u, hdr, header_num, hdr_inc); 101 header_num++; 102 } 103} 104 105static void memswp(void *buf1, void *buf2, unsigned int len) 106{ 107 char swap[200]; 108 109 assert(len <= sizeof(swap)); 110 111 memcpy(&swap, buf1, len); 112 memcpy(buf1, buf2, len); 113 memcpy(buf2, &swap, len); 114} 115 116static void hexdump(void *buffer, int len) 117{ 118 unsigned char *p = buffer; 119 int i; 120 121 for (i = 0; i < len; i++) 122 log_err("%02x", p[i]); 123 log_err("\n"); 124} 125 126/* 127 * Prepare for seperation of verify_header and checksum header 128 */ 129static inline unsigned int __hdr_size(int verify_type) 130{ 131 unsigned int len = 0; 132 133 switch (verify_type) { 134 case VERIFY_NONE: 135 case VERIFY_NULL: 136 len = 0; 137 break; 138 case VERIFY_MD5: 139 len = sizeof(struct vhdr_md5); 140 break; 141 case VERIFY_CRC64: 142 len = sizeof(struct vhdr_crc64); 143 break; 144 case VERIFY_CRC32C: 145 case VERIFY_CRC32: 146 case VERIFY_CRC32C_INTEL: 147 len = sizeof(struct vhdr_crc32); 148 break; 149 case VERIFY_CRC16: 150 len = sizeof(struct vhdr_crc16); 151 break; 152 case VERIFY_CRC7: 153 len = sizeof(struct vhdr_crc7); 154 break; 155 case VERIFY_SHA256: 156 len = sizeof(struct vhdr_sha256); 157 break; 158 case VERIFY_SHA512: 159 len = sizeof(struct vhdr_sha512); 160 break; 161 case VERIFY_META: 162 len = sizeof(struct vhdr_meta); 163 break; 164 case VERIFY_SHA1: 165 len = sizeof(struct vhdr_sha1); 166 break; 167 case VERIFY_PATTERN: 168 len = 0; 169 break; 170 default: 171 log_err("fio: unknown verify header!\n"); 172 assert(0); 173 } 174 175 return len + sizeof(struct verify_header); 176} 177 178static inline unsigned int hdr_size(struct verify_header *hdr) 179{ 180 return __hdr_size(hdr->verify_type); 181} 182 183static void *hdr_priv(struct verify_header *hdr) 184{ 185 void *priv = hdr; 186 187 return priv + sizeof(struct verify_header); 188} 189 190/* 191 * Verify container, pass info to verify handlers and allow them to 192 * pass info back in case of error 193 */ 194struct vcont { 195 /* 196 * Input 197 */ 198 struct io_u *io_u; 199 unsigned int hdr_num; 200 struct thread_data *td; 201 202 /* 203 * Output, only valid in case of error 204 */ 205 const char *name; 206 void *good_crc; 207 void *bad_crc; 208 unsigned int crc_len; 209}; 210 211static void dump_buf(char *buf, unsigned int len, unsigned long long offset, 212 const char *type, struct fio_file *f) 213{ 214 char *ptr, fname[256]; 215 int ret, fd; 216 217 ptr = strdup(f->file_name); 218 strcpy(fname, basename(ptr)); 219 220 sprintf(fname + strlen(fname), ".%llu.%s", offset, type); 221 222 fd = open(fname, O_CREAT | O_TRUNC | O_WRONLY, 0644); 223 if (fd < 0) { 224 perror("open verify buf file"); 225 return; 226 } 227 228 while (len) { 229 ret = write(fd, buf, len); 230 if (!ret) 231 break; 232 else if (ret < 0) { 233 perror("write verify buf file"); 234 break; 235 } 236 len -= ret; 237 buf += ret; 238 } 239 240 close(fd); 241 log_err(" %s data dumped as %s\n", type, fname); 242 free(ptr); 243} 244 245/* 246 * Dump the contents of the read block and re-generate the correct data 247 * and dump that too. 248 */ 249static void dump_verify_buffers(struct verify_header *hdr, struct vcont *vc) 250{ 251 struct thread_data *td = vc->td; 252 struct io_u *io_u = vc->io_u; 253 unsigned long hdr_offset; 254 struct io_u dummy; 255 void *buf; 256 257 if (!td->o.verify_dump) 258 return; 259 260 /* 261 * Dump the contents we just read off disk 262 */ 263 hdr_offset = vc->hdr_num * hdr->len; 264 265 dump_buf(io_u->buf + hdr_offset, hdr->len, io_u->offset + hdr_offset, 266 "received", vc->io_u->file); 267 268 /* 269 * Allocate a new buf and re-generate the original data 270 */ 271 buf = malloc(io_u->buflen); 272 dummy = *io_u; 273 dummy.buf = buf; 274 dummy.rand_seed = hdr->rand_seed; 275 dummy.buf_filled_len = 0; 276 277 fill_pattern_headers(td, &dummy, hdr->rand_seed, 1); 278 279 dump_buf(buf + hdr_offset, hdr->len, io_u->offset + hdr_offset, 280 "expected", vc->io_u->file); 281 free(buf); 282} 283 284static void log_verify_failure(struct verify_header *hdr, struct vcont *vc) 285{ 286 unsigned long long offset; 287 288 offset = vc->io_u->offset; 289 offset += vc->hdr_num * hdr->len; 290 log_err("%.8s: verify failed at file %s offset %llu, length %u\n", 291 vc->name, vc->io_u->file->file_name, offset, hdr->len); 292 293 if (vc->good_crc && vc->bad_crc) { 294 log_err(" Expected CRC: "); 295 hexdump(vc->good_crc, vc->crc_len); 296 log_err(" Received CRC: "); 297 hexdump(vc->bad_crc, vc->crc_len); 298 } 299 300 dump_verify_buffers(hdr, vc); 301} 302 303/* 304 * Return data area 'header_num' 305 */ 306static inline void *io_u_verify_off(struct verify_header *hdr, struct vcont *vc) 307{ 308 return vc->io_u->buf + vc->hdr_num * hdr->len + hdr_size(hdr); 309} 310 311static unsigned int hweight8(unsigned int w) 312{ 313 unsigned int res = w - ((w >> 1) & 0x55); 314 315 res = (res & 0x33) + ((res >> 2) & 0x33); 316 return (res + (res >> 4)) & 0x0F; 317} 318 319static int verify_io_u_pattern(struct verify_header *hdr, struct vcont *vc) 320{ 321 struct thread_data *td = vc->td; 322 struct io_u *io_u = vc->io_u; 323 char *buf, *pattern; 324 unsigned int header_size = __hdr_size(td->o.verify); 325 unsigned int len, mod, i, size, pattern_size; 326 327 pattern = td->o.verify_pattern; 328 pattern_size = td->o.verify_pattern_bytes; 329 if (pattern_size <= 1) 330 pattern_size = MAX_PATTERN_SIZE; 331 buf = (void *) hdr + header_size; 332 len = get_hdr_inc(td, io_u) - header_size; 333 mod = header_size % pattern_size; 334 335 for (i = 0; i < len; i += size) { 336 size = pattern_size - mod; 337 if (size > (len - i)) 338 size = len - i; 339 if (memcmp(buf + i, pattern + mod, size)) 340 /* Let the slow compare find the first mismatch byte. */ 341 break; 342 mod = 0; 343 } 344 345 for (; i < len; i++) { 346 if (buf[i] != pattern[mod]) { 347 unsigned int bits; 348 349 bits = hweight8(buf[i] ^ pattern[mod]); 350 log_err("fio: got pattern %x, wanted %x. Bad bits %d\n", 351 buf[i], pattern[mod], bits); 352 log_err("fio: bad pattern block offset %u\n", i); 353 dump_verify_buffers(hdr, vc); 354 return EILSEQ; 355 } 356 mod++; 357 if (mod == td->o.verify_pattern_bytes) 358 mod = 0; 359 } 360 361 return 0; 362} 363 364static int verify_io_u_meta(struct verify_header *hdr, struct vcont *vc) 365{ 366 struct thread_data *td = vc->td; 367 struct vhdr_meta *vh = hdr_priv(hdr); 368 struct io_u *io_u = vc->io_u; 369 int ret = EILSEQ; 370 371 dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len); 372 373 if (vh->offset == io_u->offset + vc->hdr_num * td->o.verify_interval) 374 ret = 0; 375 376 if (td->o.verify_pattern_bytes) 377 ret |= verify_io_u_pattern(hdr, vc); 378 379 if (!ret) 380 return 0; 381 382 vc->name = "meta"; 383 log_verify_failure(hdr, vc); 384 return ret; 385} 386 387static int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc) 388{ 389 void *p = io_u_verify_off(hdr, vc); 390 struct vhdr_sha512 *vh = hdr_priv(hdr); 391 uint8_t sha512[128]; 392 struct sha512_ctx sha512_ctx = { 393 .buf = sha512, 394 }; 395 396 dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", vc->io_u, hdr->len); 397 398 sha512_init(&sha512_ctx); 399 sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr)); 400 401 if (!memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512))) 402 return 0; 403 404 vc->name = "sha512"; 405 vc->good_crc = vh->sha512; 406 vc->bad_crc = sha512_ctx.buf; 407 vc->crc_len = sizeof(vh->sha512); 408 log_verify_failure(hdr, vc); 409 return EILSEQ; 410} 411 412static int verify_io_u_sha256(struct verify_header *hdr, struct vcont *vc) 413{ 414 void *p = io_u_verify_off(hdr, vc); 415 struct vhdr_sha256 *vh = hdr_priv(hdr); 416 uint8_t sha256[64]; 417 struct sha256_ctx sha256_ctx = { 418 .buf = sha256, 419 }; 420 421 dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", vc->io_u, hdr->len); 422 423 sha256_init(&sha256_ctx); 424 sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr)); 425 426 if (!memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256))) 427 return 0; 428 429 vc->name = "sha256"; 430 vc->good_crc = vh->sha256; 431 vc->bad_crc = sha256_ctx.buf; 432 vc->crc_len = sizeof(vh->sha256); 433 log_verify_failure(hdr, vc); 434 return EILSEQ; 435} 436 437static int verify_io_u_sha1(struct verify_header *hdr, struct vcont *vc) 438{ 439 void *p = io_u_verify_off(hdr, vc); 440 struct vhdr_sha1 *vh = hdr_priv(hdr); 441 uint32_t sha1[5]; 442 struct sha1_ctx sha1_ctx = { 443 .H = sha1, 444 }; 445 446 dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", vc->io_u, hdr->len); 447 448 sha1_init(&sha1_ctx); 449 sha1_update(&sha1_ctx, p, hdr->len - hdr_size(hdr)); 450 451 if (!memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1))) 452 return 0; 453 454 vc->name = "sha1"; 455 vc->good_crc = vh->sha1; 456 vc->bad_crc = sha1_ctx.H; 457 vc->crc_len = sizeof(vh->sha1); 458 log_verify_failure(hdr, vc); 459 return EILSEQ; 460} 461 462static int verify_io_u_crc7(struct verify_header *hdr, struct vcont *vc) 463{ 464 void *p = io_u_verify_off(hdr, vc); 465 struct vhdr_crc7 *vh = hdr_priv(hdr); 466 unsigned char c; 467 468 dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", vc->io_u, hdr->len); 469 470 c = crc7(p, hdr->len - hdr_size(hdr)); 471 472 if (c == vh->crc7) 473 return 0; 474 475 vc->name = "crc7"; 476 vc->good_crc = &vh->crc7; 477 vc->bad_crc = &c; 478 vc->crc_len = 1; 479 log_verify_failure(hdr, vc); 480 return EILSEQ; 481} 482 483static int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc) 484{ 485 void *p = io_u_verify_off(hdr, vc); 486 struct vhdr_crc16 *vh = hdr_priv(hdr); 487 unsigned short c; 488 489 dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len); 490 491 c = crc16(p, hdr->len - hdr_size(hdr)); 492 493 if (c == vh->crc16) 494 return 0; 495 496 vc->name = "crc16"; 497 vc->good_crc = &vh->crc16; 498 vc->bad_crc = &c; 499 vc->crc_len = 2; 500 log_verify_failure(hdr, vc); 501 return EILSEQ; 502} 503 504static int verify_io_u_crc64(struct verify_header *hdr, struct vcont *vc) 505{ 506 void *p = io_u_verify_off(hdr, vc); 507 struct vhdr_crc64 *vh = hdr_priv(hdr); 508 unsigned long long c; 509 510 dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", vc->io_u, hdr->len); 511 512 c = crc64(p, hdr->len - hdr_size(hdr)); 513 514 if (c == vh->crc64) 515 return 0; 516 517 vc->name = "crc64"; 518 vc->good_crc = &vh->crc64; 519 vc->bad_crc = &c; 520 vc->crc_len = 8; 521 log_verify_failure(hdr, vc); 522 return EILSEQ; 523} 524 525static int verify_io_u_crc32(struct verify_header *hdr, struct vcont *vc) 526{ 527 void *p = io_u_verify_off(hdr, vc); 528 struct vhdr_crc32 *vh = hdr_priv(hdr); 529 uint32_t c; 530 531 dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", vc->io_u, hdr->len); 532 533 c = crc32(p, hdr->len - hdr_size(hdr)); 534 535 if (c == vh->crc32) 536 return 0; 537 538 vc->name = "crc32"; 539 vc->good_crc = &vh->crc32; 540 vc->bad_crc = &c; 541 vc->crc_len = 4; 542 log_verify_failure(hdr, vc); 543 return EILSEQ; 544} 545 546static int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc) 547{ 548 void *p = io_u_verify_off(hdr, vc); 549 struct vhdr_crc32 *vh = hdr_priv(hdr); 550 uint32_t c; 551 552 dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len); 553 554 c = crc32c(p, hdr->len - hdr_size(hdr)); 555 556 if (c == vh->crc32) 557 return 0; 558 559 vc->name = "crc32c"; 560 vc->good_crc = &vh->crc32; 561 vc->bad_crc = &c; 562 vc->crc_len = 4; 563 log_verify_failure(hdr, vc); 564 return EILSEQ; 565} 566 567static int verify_io_u_md5(struct verify_header *hdr, struct vcont *vc) 568{ 569 void *p = io_u_verify_off(hdr, vc); 570 struct vhdr_md5 *vh = hdr_priv(hdr); 571 uint32_t hash[MD5_HASH_WORDS]; 572 struct md5_ctx md5_ctx = { 573 .hash = hash, 574 }; 575 576 dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", vc->io_u, hdr->len); 577 578 md5_init(&md5_ctx); 579 md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr)); 580 581 if (!memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash))) 582 return 0; 583 584 vc->name = "md5"; 585 vc->good_crc = vh->md5_digest; 586 vc->bad_crc = md5_ctx.hash; 587 vc->crc_len = sizeof(hash); 588 log_verify_failure(hdr, vc); 589 return EILSEQ; 590} 591 592/* 593 * Push IO verification to a separate thread 594 */ 595int verify_io_u_async(struct thread_data *td, struct io_u *io_u) 596{ 597 if (io_u->file) 598 put_file_log(td, io_u->file); 599 600 pthread_mutex_lock(&td->io_u_lock); 601 602 if (io_u->flags & IO_U_F_IN_CUR_DEPTH) { 603 td->cur_depth--; 604 io_u->flags &= ~IO_U_F_IN_CUR_DEPTH; 605 } 606 flist_del(&io_u->list); 607 flist_add_tail(&io_u->list, &td->verify_list); 608 io_u->flags |= IO_U_F_FREE_DEF; 609 pthread_mutex_unlock(&td->io_u_lock); 610 611 pthread_cond_signal(&td->verify_cond); 612 return 0; 613} 614 615static int verify_trimmed_io_u(struct thread_data *td, struct io_u *io_u) 616{ 617 static char zero_buf[1024]; 618 unsigned int this_len, len; 619 int ret = 0; 620 void *p; 621 622 if (!td->o.trim_zero) 623 return 0; 624 625 len = io_u->buflen; 626 p = io_u->buf; 627 do { 628 this_len = sizeof(zero_buf); 629 if (this_len > len) 630 this_len = len; 631 if (memcmp(p, zero_buf, this_len)) { 632 ret = EILSEQ; 633 break; 634 } 635 len -= this_len; 636 p += this_len; 637 } while (len); 638 639 if (!ret) 640 return 0; 641 642 log_err("trim: verify failed at file %s offset %llu, length %lu" 643 ", block offset %lu\n", 644 io_u->file->file_name, io_u->offset, io_u->buflen, 645 (unsigned long) (p - io_u->buf)); 646 return ret; 647} 648 649static int verify_hdr_crc(struct verify_header *hdr) 650{ 651 void *p = hdr; 652 uint32_t crc; 653 654 crc = crc32c(p, sizeof(*hdr) - sizeof(hdr->crc32)); 655 if (crc == hdr->crc32) 656 return 1; 657 658 log_err("fio: verify header crc %x, calculated %x\n", hdr->crc32, crc); 659 return 0; 660} 661 662int verify_io_u(struct thread_data *td, struct io_u *io_u) 663{ 664 struct verify_header *hdr; 665 unsigned int header_size, hdr_inc, hdr_num = 0; 666 void *p; 667 int ret; 668 669 if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ) 670 return 0; 671 if (io_u->flags & IO_U_F_TRIMMED) { 672 ret = verify_trimmed_io_u(td, io_u); 673 goto done; 674 } 675 676 hdr_inc = get_hdr_inc(td, io_u); 677 678 ret = 0; 679 for (p = io_u->buf; p < io_u->buf + io_u->buflen; 680 p += hdr_inc, hdr_num++) { 681 struct vcont vc = { 682 .io_u = io_u, 683 .hdr_num = hdr_num, 684 .td = td, 685 }; 686 687 if (ret && td->o.verify_fatal) 688 break; 689 690 header_size = __hdr_size(td->o.verify); 691 if (td->o.verify_offset) 692 memswp(p, p + td->o.verify_offset, header_size); 693 hdr = p; 694 695 if (hdr->magic != FIO_HDR_MAGIC || !verify_hdr_crc(hdr)) { 696 log_err("verify: bad magic header %x, wanted %x at file %s offset %llu, length %u\n", 697 hdr->magic, FIO_HDR_MAGIC, 698 io_u->file->file_name, 699 io_u->offset + hdr_num * hdr->len, hdr->len); 700 return EILSEQ; 701 } 702 703 switch (hdr->verify_type) { 704 case VERIFY_MD5: 705 ret = verify_io_u_md5(hdr, &vc); 706 break; 707 case VERIFY_CRC64: 708 ret = verify_io_u_crc64(hdr, &vc); 709 break; 710 case VERIFY_CRC32C: 711 case VERIFY_CRC32C_INTEL: 712 ret = verify_io_u_crc32c(hdr, &vc); 713 break; 714 case VERIFY_CRC32: 715 ret = verify_io_u_crc32(hdr, &vc); 716 break; 717 case VERIFY_CRC16: 718 ret = verify_io_u_crc16(hdr, &vc); 719 break; 720 case VERIFY_CRC7: 721 ret = verify_io_u_crc7(hdr, &vc); 722 break; 723 case VERIFY_SHA256: 724 ret = verify_io_u_sha256(hdr, &vc); 725 break; 726 case VERIFY_SHA512: 727 ret = verify_io_u_sha512(hdr, &vc); 728 break; 729 case VERIFY_META: 730 ret = verify_io_u_meta(hdr, &vc); 731 break; 732 case VERIFY_SHA1: 733 ret = verify_io_u_sha1(hdr, &vc); 734 break; 735 case VERIFY_PATTERN: 736 ret = verify_io_u_pattern(hdr, &vc); 737 break; 738 default: 739 log_err("Bad verify type %u\n", hdr->verify_type); 740 ret = EINVAL; 741 } 742 } 743 744done: 745 if (ret && td->o.verify_fatal) 746 td->terminate = 1; 747 748 return ret; 749} 750 751static void fill_meta(struct verify_header *hdr, struct thread_data *td, 752 struct io_u *io_u, unsigned int header_num) 753{ 754 struct vhdr_meta *vh = hdr_priv(hdr); 755 756 vh->thread = td->thread_number; 757 758 vh->time_sec = io_u->start_time.tv_sec; 759 vh->time_usec = io_u->start_time.tv_usec; 760 761 vh->numberio = td->io_issues[DDIR_WRITE]; 762 763 vh->offset = io_u->offset + header_num * td->o.verify_interval; 764} 765 766static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len) 767{ 768 struct vhdr_sha512 *vh = hdr_priv(hdr); 769 struct sha512_ctx sha512_ctx = { 770 .buf = vh->sha512, 771 }; 772 773 sha512_init(&sha512_ctx); 774 sha512_update(&sha512_ctx, p, len); 775} 776 777static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len) 778{ 779 struct vhdr_sha256 *vh = hdr_priv(hdr); 780 struct sha256_ctx sha256_ctx = { 781 .buf = vh->sha256, 782 }; 783 784 sha256_init(&sha256_ctx); 785 sha256_update(&sha256_ctx, p, len); 786} 787 788static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len) 789{ 790 struct vhdr_sha1 *vh = hdr_priv(hdr); 791 struct sha1_ctx sha1_ctx = { 792 .H = vh->sha1, 793 }; 794 795 sha1_init(&sha1_ctx); 796 sha1_update(&sha1_ctx, p, len); 797} 798 799static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len) 800{ 801 struct vhdr_crc7 *vh = hdr_priv(hdr); 802 803 vh->crc7 = crc7(p, len); 804} 805 806static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len) 807{ 808 struct vhdr_crc16 *vh = hdr_priv(hdr); 809 810 vh->crc16 = crc16(p, len); 811} 812 813static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len) 814{ 815 struct vhdr_crc32 *vh = hdr_priv(hdr); 816 817 vh->crc32 = crc32(p, len); 818} 819 820static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len) 821{ 822 struct vhdr_crc32 *vh = hdr_priv(hdr); 823 824 vh->crc32 = crc32c(p, len); 825} 826 827static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len) 828{ 829 struct vhdr_crc64 *vh = hdr_priv(hdr); 830 831 vh->crc64 = crc64(p, len); 832} 833 834static void fill_md5(struct verify_header *hdr, void *p, unsigned int len) 835{ 836 struct vhdr_md5 *vh = hdr_priv(hdr); 837 struct md5_ctx md5_ctx = { 838 .hash = (uint32_t *) vh->md5_digest, 839 }; 840 841 md5_init(&md5_ctx); 842 md5_update(&md5_ctx, p, len); 843} 844 845static void populate_hdr(struct thread_data *td, struct io_u *io_u, 846 struct verify_header *hdr, unsigned int header_num, 847 unsigned int header_len) 848{ 849 unsigned int data_len; 850 void *data, *p; 851 852 p = (void *) hdr; 853 854 hdr->magic = FIO_HDR_MAGIC; 855 hdr->verify_type = td->o.verify; 856 hdr->len = header_len; 857 hdr->rand_seed = io_u->rand_seed; 858 hdr->crc32 = crc32c(p, sizeof(*hdr) - sizeof(hdr->crc32)); 859 860 data_len = header_len - hdr_size(hdr); 861 862 data = p + hdr_size(hdr); 863 switch (td->o.verify) { 864 case VERIFY_MD5: 865 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n", 866 io_u, hdr->len); 867 fill_md5(hdr, data, data_len); 868 break; 869 case VERIFY_CRC64: 870 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n", 871 io_u, hdr->len); 872 fill_crc64(hdr, data, data_len); 873 break; 874 case VERIFY_CRC32C: 875 case VERIFY_CRC32C_INTEL: 876 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n", 877 io_u, hdr->len); 878 fill_crc32c(hdr, data, data_len); 879 break; 880 case VERIFY_CRC32: 881 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n", 882 io_u, hdr->len); 883 fill_crc32(hdr, data, data_len); 884 break; 885 case VERIFY_CRC16: 886 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n", 887 io_u, hdr->len); 888 fill_crc16(hdr, data, data_len); 889 break; 890 case VERIFY_CRC7: 891 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n", 892 io_u, hdr->len); 893 fill_crc7(hdr, data, data_len); 894 break; 895 case VERIFY_SHA256: 896 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n", 897 io_u, hdr->len); 898 fill_sha256(hdr, data, data_len); 899 break; 900 case VERIFY_SHA512: 901 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n", 902 io_u, hdr->len); 903 fill_sha512(hdr, data, data_len); 904 break; 905 case VERIFY_META: 906 dprint(FD_VERIFY, "fill meta io_u %p, len %u\n", 907 io_u, hdr->len); 908 fill_meta(hdr, td, io_u, header_num); 909 break; 910 case VERIFY_SHA1: 911 dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n", 912 io_u, hdr->len); 913 fill_sha1(hdr, data, data_len); 914 break; 915 case VERIFY_PATTERN: 916 /* nothing to do here */ 917 break; 918 default: 919 log_err("fio: bad verify type: %d\n", td->o.verify); 920 assert(0); 921 } 922 if (td->o.verify_offset) 923 memswp(p, p + td->o.verify_offset, hdr_size(hdr)); 924} 925 926/* 927 * fill body of io_u->buf with random data and add a header with the 928 * checksum of choice 929 */ 930void populate_verify_io_u(struct thread_data *td, struct io_u *io_u) 931{ 932 if (td->o.verify == VERIFY_NULL) 933 return; 934 935 fill_pattern_headers(td, io_u, 0, 0); 936} 937 938int get_next_verify(struct thread_data *td, struct io_u *io_u) 939{ 940 struct io_piece *ipo = NULL; 941 942 /* 943 * this io_u is from a requeue, we already filled the offsets 944 */ 945 if (io_u->file) 946 return 0; 947 948 if (!RB_EMPTY_ROOT(&td->io_hist_tree)) { 949 struct rb_node *n = rb_first(&td->io_hist_tree); 950 951 ipo = rb_entry(n, struct io_piece, rb_node); 952 rb_erase(n, &td->io_hist_tree); 953 assert(ipo->flags & IP_F_ONRB); 954 ipo->flags &= ~IP_F_ONRB; 955 } else if (!flist_empty(&td->io_hist_list)) { 956 ipo = flist_entry(td->io_hist_list.next, struct io_piece, list); 957 flist_del(&ipo->list); 958 assert(ipo->flags & IP_F_ONLIST); 959 ipo->flags &= ~IP_F_ONLIST; 960 } 961 962 if (ipo) { 963 td->io_hist_len--; 964 965 io_u->offset = ipo->offset; 966 io_u->buflen = ipo->len; 967 io_u->file = ipo->file; 968 969 if (ipo->flags & IP_F_TRIMMED) 970 io_u->flags |= IO_U_F_TRIMMED; 971 972 if (!fio_file_open(io_u->file)) { 973 int r = td_io_open_file(td, io_u->file); 974 975 if (r) { 976 dprint(FD_VERIFY, "failed file %s open\n", 977 io_u->file->file_name); 978 return 1; 979 } 980 } 981 982 get_file(ipo->file); 983 assert(fio_file_open(io_u->file)); 984 io_u->ddir = DDIR_READ; 985 io_u->xfer_buf = io_u->buf; 986 io_u->xfer_buflen = io_u->buflen; 987 988 remove_trim_entry(td, ipo); 989 free(ipo); 990 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u); 991 return 0; 992 } 993 994 dprint(FD_VERIFY, "get_next_verify: empty\n"); 995 return 1; 996} 997 998static void *verify_async_thread(void *data) 999{ 1000 struct thread_data *td = data; 1001 struct io_u *io_u; 1002 int ret = 0; 1003 1004 if (td->o.verify_cpumask_set && 1005 fio_setaffinity(td->pid, td->o.verify_cpumask)) { 1006 log_err("fio: failed setting verify thread affinity\n"); 1007 goto done; 1008 } 1009 1010 do { 1011 FLIST_HEAD(list); 1012 1013 read_barrier(); 1014 if (td->verify_thread_exit) 1015 break; 1016 1017 pthread_mutex_lock(&td->io_u_lock); 1018 1019 while (flist_empty(&td->verify_list) && 1020 !td->verify_thread_exit) { 1021 ret = pthread_cond_wait(&td->verify_cond, 1022 &td->io_u_lock); 1023 if (ret) { 1024 pthread_mutex_unlock(&td->io_u_lock); 1025 break; 1026 } 1027 } 1028 1029 flist_splice_init(&td->verify_list, &list); 1030 pthread_mutex_unlock(&td->io_u_lock); 1031 1032 if (flist_empty(&list)) 1033 continue; 1034 1035 while (!flist_empty(&list)) { 1036 io_u = flist_entry(list.next, struct io_u, list); 1037 flist_del_init(&io_u->list); 1038 1039 ret = verify_io_u(td, io_u); 1040 put_io_u(td, io_u); 1041 if (!ret) 1042 continue; 1043 if (td->o.continue_on_error & ERROR_TYPE_VERIFY && 1044 td_non_fatal_error(ret)) { 1045 update_error_count(td, ret); 1046 td_clear_error(td); 1047 ret = 0; 1048 } 1049 } 1050 } while (!ret); 1051 1052 if (ret) { 1053 td_verror(td, ret, "async_verify"); 1054 if (td->o.verify_fatal) 1055 td->terminate = 1; 1056 } 1057 1058done: 1059 pthread_mutex_lock(&td->io_u_lock); 1060 td->nr_verify_threads--; 1061 pthread_mutex_unlock(&td->io_u_lock); 1062 1063 pthread_cond_signal(&td->free_cond); 1064 return NULL; 1065} 1066 1067int verify_async_init(struct thread_data *td) 1068{ 1069 int i, ret; 1070 pthread_attr_t attr; 1071 1072 pthread_attr_init(&attr); 1073 pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN); 1074 1075 td->verify_thread_exit = 0; 1076 1077 td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async); 1078 for (i = 0; i < td->o.verify_async; i++) { 1079 ret = pthread_create(&td->verify_threads[i], &attr, 1080 verify_async_thread, td); 1081 if (ret) { 1082 log_err("fio: async verify creation failed: %s\n", 1083 strerror(ret)); 1084 break; 1085 } 1086 ret = pthread_detach(td->verify_threads[i]); 1087 if (ret) { 1088 log_err("fio: async verify thread detach failed: %s\n", 1089 strerror(ret)); 1090 break; 1091 } 1092 td->nr_verify_threads++; 1093 } 1094 1095 pthread_attr_destroy(&attr); 1096 1097 if (i != td->o.verify_async) { 1098 log_err("fio: only %d verify threads started, exiting\n", i); 1099 td->verify_thread_exit = 1; 1100 write_barrier(); 1101 pthread_cond_broadcast(&td->verify_cond); 1102 return 1; 1103 } 1104 1105 return 0; 1106} 1107 1108void verify_async_exit(struct thread_data *td) 1109{ 1110 td->verify_thread_exit = 1; 1111 write_barrier(); 1112 pthread_cond_broadcast(&td->verify_cond); 1113 1114 pthread_mutex_lock(&td->io_u_lock); 1115 1116 while (td->nr_verify_threads) 1117 pthread_cond_wait(&td->free_cond, &td->io_u_lock); 1118 1119 pthread_mutex_unlock(&td->io_u_lock); 1120 free(td->verify_threads); 1121 td->verify_threads = NULL; 1122} 1123