verify.c revision 4aae38a645f3315ca2f8cc57a7ec4110ba7d8319
1/* 2 * IO verification helpers 3 */ 4#include <unistd.h> 5#include <fcntl.h> 6#include <string.h> 7#include <assert.h> 8#include <pthread.h> 9 10#include "fio.h" 11#include "verify.h" 12#include "smalloc.h" 13#include "trim.h" 14#include "lib/rand.h" 15 16#include "crc/md5.h" 17#include "crc/crc64.h" 18#include "crc/crc32.h" 19#include "crc/crc32c.h" 20#include "crc/crc16.h" 21#include "crc/crc7.h" 22#include "crc/sha256.h" 23#include "crc/sha512.h" 24#include "crc/sha1.h" 25 26static void populate_hdr(struct thread_data *td, struct io_u *io_u, 27 struct verify_header *hdr, unsigned int header_num, 28 unsigned int header_len); 29 30void fill_pattern(struct thread_data *td, void *p, unsigned int len, struct io_u *io_u, unsigned long seed, int use_seed) 31{ 32 switch (td->o.verify_pattern_bytes) { 33 case 0: 34 dprint(FD_VERIFY, "fill random bytes len=%u\n", len); 35 if (use_seed) 36 __fill_random_buf(p, len, seed); 37 else 38 io_u->rand_seed = fill_random_buf(p, len); 39 break; 40 case 1: 41 if (io_u->buf_filled_len >= len) { 42 dprint(FD_VERIFY, "using already filled verify pattern b=0 len=%u\n", len); 43 return; 44 } 45 dprint(FD_VERIFY, "fill verify pattern b=0 len=%u\n", len); 46 memset(p, td->o.verify_pattern[0], len); 47 io_u->buf_filled_len = len; 48 break; 49 default: { 50 unsigned int i = 0, size = 0; 51 unsigned char *b = p; 52 53 if (io_u->buf_filled_len >= len) { 54 dprint(FD_VERIFY, "using already filled verify pattern b=%d len=%u\n", 55 td->o.verify_pattern_bytes, len); 56 return; 57 } 58 dprint(FD_VERIFY, "fill verify pattern b=%d len=%u\n", 59 td->o.verify_pattern_bytes, len); 60 61 while (i < len) { 62 size = td->o.verify_pattern_bytes; 63 if (size > (len - i)) 64 size = len - i; 65 memcpy(b+i, td->o.verify_pattern, size); 66 i += size; 67 } 68 io_u->buf_filled_len = len; 69 break; 70 } 71 } 72} 73 74static void fill_pattern_headers(struct thread_data *td, struct io_u *io_u, 75 unsigned long seed, int use_seed) 76{ 77 unsigned int hdr_inc, header_num; 78 struct verify_header *hdr; 79 void *p = io_u->buf; 80 81 fill_pattern(td, p, io_u->buflen, io_u, seed, use_seed); 82 83 hdr_inc = io_u->buflen; 84 if (td->o.verify_interval) 85 hdr_inc = td->o.verify_interval; 86 87 header_num = 0; 88 for (; p < io_u->buf + io_u->buflen; p += hdr_inc) { 89 hdr = p; 90 populate_hdr(td, io_u, hdr, header_num, hdr_inc); 91 header_num++; 92 } 93} 94 95static void memswp(void *buf1, void *buf2, unsigned int len) 96{ 97 char swap[200]; 98 99 assert(len <= sizeof(swap)); 100 101 memcpy(&swap, buf1, len); 102 memcpy(buf1, buf2, len); 103 memcpy(buf2, &swap, len); 104} 105 106static void hexdump(void *buffer, int len) 107{ 108 unsigned char *p = buffer; 109 int i; 110 111 for (i = 0; i < len; i++) 112 log_err("%02x", p[i]); 113 log_err("\n"); 114} 115 116/* 117 * Prepare for seperation of verify_header and checksum header 118 */ 119static inline unsigned int __hdr_size(int verify_type) 120{ 121 unsigned int len = 0; 122 123 switch (verify_type) { 124 case VERIFY_NONE: 125 case VERIFY_NULL: 126 len = 0; 127 break; 128 case VERIFY_MD5: 129 len = sizeof(struct vhdr_md5); 130 break; 131 case VERIFY_CRC64: 132 len = sizeof(struct vhdr_crc64); 133 break; 134 case VERIFY_CRC32C: 135 case VERIFY_CRC32: 136 case VERIFY_CRC32C_INTEL: 137 len = sizeof(struct vhdr_crc32); 138 break; 139 case VERIFY_CRC16: 140 len = sizeof(struct vhdr_crc16); 141 break; 142 case VERIFY_CRC7: 143 len = sizeof(struct vhdr_crc7); 144 break; 145 case VERIFY_SHA256: 146 len = sizeof(struct vhdr_sha256); 147 break; 148 case VERIFY_SHA512: 149 len = sizeof(struct vhdr_sha512); 150 break; 151 case VERIFY_META: 152 len = sizeof(struct vhdr_meta); 153 break; 154 case VERIFY_SHA1: 155 len = sizeof(struct vhdr_sha1); 156 break; 157 default: 158 log_err("fio: unknown verify header!\n"); 159 assert(0); 160 } 161 162 return len + sizeof(struct verify_header); 163} 164 165static inline unsigned int hdr_size(struct verify_header *hdr) 166{ 167 return __hdr_size(hdr->verify_type); 168} 169 170static void *hdr_priv(struct verify_header *hdr) 171{ 172 void *priv = hdr; 173 174 return priv + sizeof(struct verify_header); 175} 176 177/* 178 * Verify container, pass info to verify handlers and allow them to 179 * pass info back in case of error 180 */ 181struct vcont { 182 /* 183 * Input 184 */ 185 struct io_u *io_u; 186 unsigned int hdr_num; 187 struct thread_data *td; 188 189 /* 190 * Output, only valid in case of error 191 */ 192 const char *name; 193 void *good_crc; 194 void *bad_crc; 195 unsigned int crc_len; 196}; 197 198static void dump_buf(char *buf, unsigned int len, unsigned long long offset, 199 const char *type, struct fio_file *f) 200{ 201 char fname[256]; 202 int ret, fd; 203 204 strcpy(fname, f->file_name); 205 basename(fname); 206 207 sprintf(fname + strlen(fname), ".%llu.%s", offset, type); 208 209 fd = open(fname, O_CREAT | O_TRUNC | O_WRONLY, 0644); 210 if (fd < 0) { 211 perror("open verify buf file"); 212 return; 213 } 214 215 while (len) { 216 ret = write(fd, buf, len); 217 if (!ret) 218 break; 219 else if (ret < 0) { 220 perror("write verify buf file"); 221 break; 222 } 223 len -= ret; 224 buf += ret; 225 } 226 227 close(fd); 228 log_err(" %s data dumped as %s\n", type, fname); 229} 230 231/* 232 * Dump the contents of the read block and re-generate the correct data 233 * and dump that too. 234 */ 235static void dump_verify_buffers(struct verify_header *hdr, struct vcont *vc) 236{ 237 struct thread_data *td = vc->td; 238 struct io_u *io_u = vc->io_u; 239 unsigned long hdr_offset; 240 struct io_u dummy; 241 void *buf; 242 243 /* 244 * Dump the contents we just read off disk 245 */ 246 hdr_offset = vc->hdr_num * hdr->len; 247 248 dump_buf(io_u->buf + hdr_offset, hdr->len, io_u->offset + hdr_offset, 249 "received", vc->io_u->file); 250 251 /* 252 * Allocate a new buf and re-generate the original data 253 */ 254 buf = malloc(io_u->buflen); 255 dummy = *io_u; 256 dummy.buf = buf; 257 dummy.rand_seed = hdr->rand_seed; 258 259 fill_pattern_headers(td, &dummy, hdr->rand_seed, 1); 260 261 dump_buf(buf + hdr_offset, hdr->len, io_u->offset + hdr_offset, 262 "expected", vc->io_u->file); 263 free(buf); 264} 265 266static void log_verify_failure(struct verify_header *hdr, struct vcont *vc) 267{ 268 unsigned long long offset; 269 270 offset = vc->io_u->offset; 271 offset += vc->hdr_num * hdr->len; 272 log_err("%.8s: verify failed at file %s offset %llu, length %u\n", 273 vc->name, vc->io_u->file->file_name, offset, hdr->len); 274 275 if (vc->good_crc && vc->bad_crc) { 276 log_err(" Expected CRC: "); 277 hexdump(vc->good_crc, vc->crc_len); 278 log_err(" Received CRC: "); 279 hexdump(vc->bad_crc, vc->crc_len); 280 } 281 282 dump_verify_buffers(hdr, vc); 283} 284 285/* 286 * Return data area 'header_num' 287 */ 288static inline void *io_u_verify_off(struct verify_header *hdr, struct vcont *vc) 289{ 290 return vc->io_u->buf + vc->hdr_num * hdr->len + hdr_size(hdr); 291} 292 293static int verify_io_u_meta(struct verify_header *hdr, struct thread_data *td, 294 struct vcont *vc) 295{ 296 struct vhdr_meta *vh = hdr_priv(hdr); 297 struct io_u *io_u = vc->io_u; 298 299 dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len); 300 301 if (vh->offset == io_u->offset + vc->hdr_num * td->o.verify_interval) 302 return 0; 303 304 vc->name = "meta"; 305 log_verify_failure(hdr, vc); 306 return EILSEQ; 307} 308 309static int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc) 310{ 311 void *p = io_u_verify_off(hdr, vc); 312 struct vhdr_sha512 *vh = hdr_priv(hdr); 313 uint8_t sha512[128]; 314 struct sha512_ctx sha512_ctx = { 315 .buf = sha512, 316 }; 317 318 dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", vc->io_u, hdr->len); 319 320 sha512_init(&sha512_ctx); 321 sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr)); 322 323 if (!memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512))) 324 return 0; 325 326 vc->name = "sha512"; 327 vc->good_crc = vh->sha512; 328 vc->bad_crc = sha512_ctx.buf; 329 vc->crc_len = sizeof(vh->sha512); 330 log_verify_failure(hdr, vc); 331 return EILSEQ; 332} 333 334static int verify_io_u_sha256(struct verify_header *hdr, struct vcont *vc) 335{ 336 void *p = io_u_verify_off(hdr, vc); 337 struct vhdr_sha256 *vh = hdr_priv(hdr); 338 uint8_t sha256[64]; 339 struct sha256_ctx sha256_ctx = { 340 .buf = sha256, 341 }; 342 343 dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", vc->io_u, hdr->len); 344 345 sha256_init(&sha256_ctx); 346 sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr)); 347 348 if (!memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256))) 349 return 0; 350 351 vc->name = "sha256"; 352 vc->good_crc = vh->sha256; 353 vc->bad_crc = sha256_ctx.buf; 354 vc->crc_len = sizeof(vh->sha256); 355 log_verify_failure(hdr, vc); 356 return EILSEQ; 357} 358 359static int verify_io_u_sha1(struct verify_header *hdr, struct vcont *vc) 360{ 361 void *p = io_u_verify_off(hdr, vc); 362 struct vhdr_sha1 *vh = hdr_priv(hdr); 363 uint32_t sha1[5]; 364 struct sha1_ctx sha1_ctx = { 365 .H = sha1, 366 }; 367 368 dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", vc->io_u, hdr->len); 369 370 sha1_init(&sha1_ctx); 371 sha1_update(&sha1_ctx, p, hdr->len - hdr_size(hdr)); 372 373 if (!memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1))) 374 return 0; 375 376 vc->name = "sha1"; 377 vc->good_crc = vh->sha1; 378 vc->bad_crc = sha1_ctx.H; 379 vc->crc_len = sizeof(vh->sha1); 380 log_verify_failure(hdr, vc); 381 return EILSEQ; 382} 383 384static int verify_io_u_crc7(struct verify_header *hdr, struct vcont *vc) 385{ 386 void *p = io_u_verify_off(hdr, vc); 387 struct vhdr_crc7 *vh = hdr_priv(hdr); 388 unsigned char c; 389 390 dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", vc->io_u, hdr->len); 391 392 c = crc7(p, hdr->len - hdr_size(hdr)); 393 394 if (c == vh->crc7) 395 return 0; 396 397 vc->name = "crc7"; 398 vc->good_crc = &vh->crc7; 399 vc->bad_crc = &c; 400 vc->crc_len = 1; 401 log_verify_failure(hdr, vc); 402 return EILSEQ; 403} 404 405static int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc) 406{ 407 void *p = io_u_verify_off(hdr, vc); 408 struct vhdr_crc16 *vh = hdr_priv(hdr); 409 unsigned short c; 410 411 dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len); 412 413 c = crc16(p, hdr->len - hdr_size(hdr)); 414 415 if (c == vh->crc16) 416 return 0; 417 418 vc->name = "crc16"; 419 vc->good_crc = &vh->crc16; 420 vc->bad_crc = &c; 421 vc->crc_len = 2; 422 log_verify_failure(hdr, vc); 423 return EILSEQ; 424} 425 426static int verify_io_u_crc64(struct verify_header *hdr, struct vcont *vc) 427{ 428 void *p = io_u_verify_off(hdr, vc); 429 struct vhdr_crc64 *vh = hdr_priv(hdr); 430 unsigned long long c; 431 432 dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", vc->io_u, hdr->len); 433 434 c = crc64(p, hdr->len - hdr_size(hdr)); 435 436 if (c == vh->crc64) 437 return 0; 438 439 vc->name = "crc64"; 440 vc->good_crc = &vh->crc64; 441 vc->bad_crc = &c; 442 vc->crc_len = 8; 443 log_verify_failure(hdr, vc); 444 return EILSEQ; 445} 446 447static int verify_io_u_crc32(struct verify_header *hdr, struct vcont *vc) 448{ 449 void *p = io_u_verify_off(hdr, vc); 450 struct vhdr_crc32 *vh = hdr_priv(hdr); 451 uint32_t c; 452 453 dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", vc->io_u, hdr->len); 454 455 c = crc32(p, hdr->len - hdr_size(hdr)); 456 457 if (c == vh->crc32) 458 return 0; 459 460 vc->name = "crc32"; 461 vc->good_crc = &vh->crc32; 462 vc->bad_crc = &c; 463 vc->crc_len = 4; 464 log_verify_failure(hdr, vc); 465 return EILSEQ; 466} 467 468static int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc) 469{ 470 void *p = io_u_verify_off(hdr, vc); 471 struct vhdr_crc32 *vh = hdr_priv(hdr); 472 uint32_t c; 473 474 dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len); 475 476 if (hdr->verify_type == VERIFY_CRC32C_INTEL) 477 c = crc32c_intel(p, hdr->len - hdr_size(hdr)); 478 else 479 c = crc32c(p, hdr->len - hdr_size(hdr)); 480 481 if (c == vh->crc32) 482 return 0; 483 484 vc->name = "crc32c"; 485 vc->good_crc = &vh->crc32; 486 vc->bad_crc = &c; 487 vc->crc_len = 4; 488 log_verify_failure(hdr, vc); 489 return EILSEQ; 490} 491 492static int verify_io_u_md5(struct verify_header *hdr, struct vcont *vc) 493{ 494 void *p = io_u_verify_off(hdr, vc); 495 struct vhdr_md5 *vh = hdr_priv(hdr); 496 uint32_t hash[MD5_HASH_WORDS]; 497 struct md5_ctx md5_ctx = { 498 .hash = hash, 499 }; 500 501 dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", vc->io_u, hdr->len); 502 503 md5_init(&md5_ctx); 504 md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr)); 505 506 if (!memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash))) 507 return 0; 508 509 vc->name = "md5"; 510 vc->good_crc = vh->md5_digest; 511 vc->bad_crc = md5_ctx.hash; 512 vc->crc_len = sizeof(hash); 513 log_verify_failure(hdr, vc); 514 return EILSEQ; 515} 516 517static unsigned int hweight8(unsigned int w) 518{ 519 unsigned int res = w - ((w >> 1) & 0x55); 520 521 res = (res & 0x33) + ((res >> 2) & 0x33); 522 return (res + (res >> 4)) & 0x0F; 523} 524 525int verify_io_u_pattern(char *pattern, unsigned long pattern_size, 526 char *buf, unsigned int len, unsigned int mod) 527{ 528 unsigned int i; 529 530 for (i = 0; i < len; i++) { 531 if (buf[i] != pattern[mod]) { 532 unsigned int bits; 533 534 bits = hweight8(buf[i] ^ pattern[mod]); 535 log_err("fio: got pattern %x, wanted %x. Bad bits %d\n", 536 buf[i], pattern[mod], bits); 537 log_err("fio: bad pattern block offset %u\n", i); 538 return EILSEQ; 539 } 540 mod++; 541 if (mod == pattern_size) 542 mod = 0; 543 } 544 545 return 0; 546} 547 548/* 549 * Push IO verification to a separate thread 550 */ 551int verify_io_u_async(struct thread_data *td, struct io_u *io_u) 552{ 553 if (io_u->file) 554 put_file_log(td, io_u->file); 555 556 io_u->file = NULL; 557 558 pthread_mutex_lock(&td->io_u_lock); 559 560 if (io_u->flags & IO_U_F_IN_CUR_DEPTH) { 561 td->cur_depth--; 562 io_u->flags &= ~IO_U_F_IN_CUR_DEPTH; 563 } 564 flist_del(&io_u->list); 565 flist_add_tail(&io_u->list, &td->verify_list); 566 io_u->flags |= IO_U_F_FREE_DEF; 567 pthread_mutex_unlock(&td->io_u_lock); 568 569 pthread_cond_signal(&td->verify_cond); 570 return 0; 571} 572 573static int verify_trimmed_io_u(struct thread_data *td, struct io_u *io_u) 574{ 575 static char zero_buf[1024]; 576 unsigned int this_len, len; 577 int ret = 0; 578 void *p; 579 580 if (!td->o.trim_zero) 581 return 0; 582 583 len = io_u->buflen; 584 p = io_u->buf; 585 do { 586 this_len = sizeof(zero_buf); 587 if (this_len > len) 588 this_len = len; 589 if (memcmp(p, zero_buf, this_len)) { 590 ret = EILSEQ; 591 break; 592 } 593 len -= this_len; 594 p += this_len; 595 } while (len); 596 597 if (!ret) 598 return 0; 599 600 log_err("trim: verify failed at file %s offset %llu, length %lu" 601 ", block offset %lu\n", 602 io_u->file->file_name, io_u->offset, io_u->buflen, 603 (unsigned long) (p - io_u->buf)); 604 return ret; 605} 606 607int verify_io_u(struct thread_data *td, struct io_u *io_u) 608{ 609 struct verify_header *hdr; 610 unsigned int hdr_size, hdr_inc, hdr_num = 0; 611 void *p; 612 int ret; 613 614 if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ) 615 return 0; 616 if (io_u->flags & IO_U_F_TRIMMED) { 617 ret = verify_trimmed_io_u(td, io_u); 618 goto done; 619 } 620 621 hdr_inc = io_u->buflen; 622 if (td->o.verify_interval) 623 hdr_inc = td->o.verify_interval; 624 625 ret = 0; 626 for (p = io_u->buf; p < io_u->buf + io_u->buflen; 627 p += hdr_inc, hdr_num++) { 628 struct vcont vc = { 629 .io_u = io_u, 630 .hdr_num = hdr_num, 631 .td = td, 632 }; 633 634 if (ret && td->o.verify_fatal) 635 break; 636 637 hdr_size = __hdr_size(td->o.verify); 638 if (td->o.verify_offset) 639 memswp(p, p + td->o.verify_offset, hdr_size); 640 hdr = p; 641 642 if (hdr->fio_magic != FIO_HDR_MAGIC) { 643 log_err("verify: bad magic header %x, wanted %x at file %s offset %llu, length %u\n", 644 hdr->fio_magic, FIO_HDR_MAGIC, 645 io_u->file->file_name, 646 io_u->offset + hdr_num * hdr->len, hdr->len); 647 return EILSEQ; 648 } 649 650 if (td->o.verify_pattern_bytes) { 651 dprint(FD_VERIFY, "pattern verify io_u %p, len %u\n", 652 io_u, hdr->len); 653 ret = verify_io_u_pattern(td->o.verify_pattern, 654 td->o.verify_pattern_bytes, 655 p + hdr_size, 656 hdr_inc - hdr_size, 657 hdr_size % td->o.verify_pattern_bytes); 658 659 if (ret) { 660 log_err("pattern: verify failed at file %s offset %llu, length %u\n", 661 io_u->file->file_name, 662 io_u->offset + hdr_num * hdr->len, 663 hdr->len); 664 } 665 666 /* 667 * Also verify the meta data, if applicable 668 */ 669 if (hdr->verify_type == VERIFY_META) 670 ret |= verify_io_u_meta(hdr, td, &vc); 671 continue; 672 } 673 674 switch (hdr->verify_type) { 675 case VERIFY_MD5: 676 ret = verify_io_u_md5(hdr, &vc); 677 break; 678 case VERIFY_CRC64: 679 ret = verify_io_u_crc64(hdr, &vc); 680 break; 681 case VERIFY_CRC32C: 682 case VERIFY_CRC32C_INTEL: 683 ret = verify_io_u_crc32c(hdr, &vc); 684 break; 685 case VERIFY_CRC32: 686 ret = verify_io_u_crc32(hdr, &vc); 687 break; 688 case VERIFY_CRC16: 689 ret = verify_io_u_crc16(hdr, &vc); 690 break; 691 case VERIFY_CRC7: 692 ret = verify_io_u_crc7(hdr, &vc); 693 break; 694 case VERIFY_SHA256: 695 ret = verify_io_u_sha256(hdr, &vc); 696 break; 697 case VERIFY_SHA512: 698 ret = verify_io_u_sha512(hdr, &vc); 699 break; 700 case VERIFY_META: 701 ret = verify_io_u_meta(hdr, td, &vc); 702 break; 703 case VERIFY_SHA1: 704 ret = verify_io_u_sha1(hdr, &vc); 705 break; 706 default: 707 log_err("Bad verify type %u\n", hdr->verify_type); 708 ret = EINVAL; 709 } 710 } 711 712done: 713 if (ret && td->o.verify_fatal) 714 td->terminate = 1; 715 716 return ret; 717} 718 719static void fill_meta(struct verify_header *hdr, struct thread_data *td, 720 struct io_u *io_u, unsigned int header_num) 721{ 722 struct vhdr_meta *vh = hdr_priv(hdr); 723 724 vh->thread = td->thread_number; 725 726 vh->time_sec = io_u->start_time.tv_sec; 727 vh->time_usec = io_u->start_time.tv_usec; 728 729 vh->numberio = td->io_issues[DDIR_WRITE]; 730 731 vh->offset = io_u->offset + header_num * td->o.verify_interval; 732} 733 734static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len) 735{ 736 struct vhdr_sha512 *vh = hdr_priv(hdr); 737 struct sha512_ctx sha512_ctx = { 738 .buf = vh->sha512, 739 }; 740 741 sha512_init(&sha512_ctx); 742 sha512_update(&sha512_ctx, p, len); 743} 744 745static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len) 746{ 747 struct vhdr_sha256 *vh = hdr_priv(hdr); 748 struct sha256_ctx sha256_ctx = { 749 .buf = vh->sha256, 750 }; 751 752 sha256_init(&sha256_ctx); 753 sha256_update(&sha256_ctx, p, len); 754} 755 756static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len) 757{ 758 struct vhdr_sha1 *vh = hdr_priv(hdr); 759 struct sha1_ctx sha1_ctx = { 760 .H = vh->sha1, 761 }; 762 763 sha1_init(&sha1_ctx); 764 sha1_update(&sha1_ctx, p, len); 765} 766 767static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len) 768{ 769 struct vhdr_crc7 *vh = hdr_priv(hdr); 770 771 vh->crc7 = crc7(p, len); 772} 773 774static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len) 775{ 776 struct vhdr_crc16 *vh = hdr_priv(hdr); 777 778 vh->crc16 = crc16(p, len); 779} 780 781static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len) 782{ 783 struct vhdr_crc32 *vh = hdr_priv(hdr); 784 785 vh->crc32 = crc32(p, len); 786} 787 788static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len) 789{ 790 struct vhdr_crc32 *vh = hdr_priv(hdr); 791 792 if (hdr->verify_type == VERIFY_CRC32C_INTEL) 793 vh->crc32 = crc32c_intel(p, len); 794 else 795 vh->crc32 = crc32c(p, len); 796} 797 798static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len) 799{ 800 struct vhdr_crc64 *vh = hdr_priv(hdr); 801 802 vh->crc64 = crc64(p, len); 803} 804 805static void fill_md5(struct verify_header *hdr, void *p, unsigned int len) 806{ 807 struct vhdr_md5 *vh = hdr_priv(hdr); 808 struct md5_ctx md5_ctx = { 809 .hash = (uint32_t *) vh->md5_digest, 810 }; 811 812 md5_init(&md5_ctx); 813 md5_update(&md5_ctx, p, len); 814} 815 816static void populate_hdr(struct thread_data *td, struct io_u *io_u, 817 struct verify_header *hdr, unsigned int header_num, 818 unsigned int header_len) 819{ 820 unsigned int data_len; 821 void *data, *p; 822 823 p = (void *) hdr; 824 825 hdr->fio_magic = FIO_HDR_MAGIC; 826 hdr->len = header_len; 827 hdr->verify_type = td->o.verify; 828 hdr->rand_seed = io_u->rand_seed; 829 data_len = header_len - hdr_size(hdr); 830 831 data = p + hdr_size(hdr); 832 switch (td->o.verify) { 833 case VERIFY_MD5: 834 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n", 835 io_u, hdr->len); 836 fill_md5(hdr, data, data_len); 837 break; 838 case VERIFY_CRC64: 839 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n", 840 io_u, hdr->len); 841 fill_crc64(hdr, data, data_len); 842 break; 843 case VERIFY_CRC32C: 844 case VERIFY_CRC32C_INTEL: 845 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n", 846 io_u, hdr->len); 847 fill_crc32c(hdr, data, data_len); 848 break; 849 case VERIFY_CRC32: 850 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n", 851 io_u, hdr->len); 852 fill_crc32(hdr, data, data_len); 853 break; 854 case VERIFY_CRC16: 855 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n", 856 io_u, hdr->len); 857 fill_crc16(hdr, data, data_len); 858 break; 859 case VERIFY_CRC7: 860 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n", 861 io_u, hdr->len); 862 fill_crc7(hdr, data, data_len); 863 break; 864 case VERIFY_SHA256: 865 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n", 866 io_u, hdr->len); 867 fill_sha256(hdr, data, data_len); 868 break; 869 case VERIFY_SHA512: 870 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n", 871 io_u, hdr->len); 872 fill_sha512(hdr, data, data_len); 873 break; 874 case VERIFY_META: 875 dprint(FD_VERIFY, "fill meta io_u %p, len %u\n", 876 io_u, hdr->len); 877 fill_meta(hdr, td, io_u, header_num); 878 break; 879 case VERIFY_SHA1: 880 dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n", 881 io_u, hdr->len); 882 fill_sha1(hdr, data, data_len); 883 break; 884 default: 885 log_err("fio: bad verify type: %d\n", td->o.verify); 886 assert(0); 887 } 888 if (td->o.verify_offset) 889 memswp(p, p + td->o.verify_offset, hdr_size(hdr)); 890} 891 892/* 893 * fill body of io_u->buf with random data and add a header with the 894 * checksum of choice 895 */ 896void populate_verify_io_u(struct thread_data *td, struct io_u *io_u) 897{ 898 if (td->o.verify == VERIFY_NULL) 899 return; 900 901 fill_pattern_headers(td, io_u, 0, 0); 902} 903 904int get_next_verify(struct thread_data *td, struct io_u *io_u) 905{ 906 struct io_piece *ipo = NULL; 907 908 /* 909 * this io_u is from a requeue, we already filled the offsets 910 */ 911 if (io_u->file) 912 return 0; 913 914 if (!RB_EMPTY_ROOT(&td->io_hist_tree)) { 915 struct rb_node *n = rb_first(&td->io_hist_tree); 916 917 ipo = rb_entry(n, struct io_piece, rb_node); 918 rb_erase(n, &td->io_hist_tree); 919 assert(ipo->flags & IP_F_ONRB); 920 ipo->flags &= ~IP_F_ONRB; 921 } else if (!flist_empty(&td->io_hist_list)) { 922 ipo = flist_entry(td->io_hist_list.next, struct io_piece, list); 923 flist_del(&ipo->list); 924 assert(ipo->flags & IP_F_ONLIST); 925 ipo->flags &= ~IP_F_ONLIST; 926 } 927 928 if (ipo) { 929 td->io_hist_len--; 930 931 io_u->offset = ipo->offset; 932 io_u->buflen = ipo->len; 933 io_u->file = ipo->file; 934 935 if (ipo->flags & IP_F_TRIMMED) 936 io_u->flags |= IO_U_F_TRIMMED; 937 938 if (!fio_file_open(io_u->file)) { 939 int r = td_io_open_file(td, io_u->file); 940 941 if (r) { 942 dprint(FD_VERIFY, "failed file %s open\n", 943 io_u->file->file_name); 944 return 1; 945 } 946 } 947 948 get_file(ipo->file); 949 assert(fio_file_open(io_u->file)); 950 io_u->ddir = DDIR_READ; 951 io_u->xfer_buf = io_u->buf; 952 io_u->xfer_buflen = io_u->buflen; 953 954 remove_trim_entry(td, ipo); 955 free(ipo); 956 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u); 957 return 0; 958 } 959 960 dprint(FD_VERIFY, "get_next_verify: empty\n"); 961 return 1; 962} 963 964static void *verify_async_thread(void *data) 965{ 966 struct thread_data *td = data; 967 struct io_u *io_u; 968 int ret = 0; 969 970 if (td->o.verify_cpumask_set && 971 fio_setaffinity(td->pid, td->o.verify_cpumask)) { 972 log_err("fio: failed setting verify thread affinity\n"); 973 goto done; 974 } 975 976 do { 977 FLIST_HEAD(list); 978 979 read_barrier(); 980 if (td->verify_thread_exit) 981 break; 982 983 pthread_mutex_lock(&td->io_u_lock); 984 985 while (flist_empty(&td->verify_list) && 986 !td->verify_thread_exit) { 987 ret = pthread_cond_wait(&td->verify_cond, 988 &td->io_u_lock); 989 if (ret) { 990 pthread_mutex_unlock(&td->io_u_lock); 991 break; 992 } 993 } 994 995 flist_splice_init(&td->verify_list, &list); 996 pthread_mutex_unlock(&td->io_u_lock); 997 998 if (flist_empty(&list)) 999 continue; 1000 1001 while (!flist_empty(&list)) { 1002 io_u = flist_entry(list.next, struct io_u, list); 1003 flist_del_init(&io_u->list); 1004 1005 ret = verify_io_u(td, io_u); 1006 put_io_u(td, io_u); 1007 if (!ret) 1008 continue; 1009 if (td->o.continue_on_error && 1010 td_non_fatal_error(ret)) { 1011 update_error_count(td, ret); 1012 td_clear_error(td); 1013 ret = 0; 1014 } 1015 } 1016 } while (!ret); 1017 1018 if (ret) { 1019 td_verror(td, ret, "async_verify"); 1020 if (td->o.verify_fatal) 1021 td->terminate = 1; 1022 } 1023 1024done: 1025 pthread_mutex_lock(&td->io_u_lock); 1026 td->nr_verify_threads--; 1027 pthread_mutex_unlock(&td->io_u_lock); 1028 1029 pthread_cond_signal(&td->free_cond); 1030 return NULL; 1031} 1032 1033int verify_async_init(struct thread_data *td) 1034{ 1035 int i, ret; 1036 pthread_attr_t attr; 1037 1038 pthread_attr_init(&attr); 1039 pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN); 1040 1041 td->verify_thread_exit = 0; 1042 1043 td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async); 1044 for (i = 0; i < td->o.verify_async; i++) { 1045 ret = pthread_create(&td->verify_threads[i], &attr, 1046 verify_async_thread, td); 1047 if (ret) { 1048 log_err("fio: async verify creation failed: %s\n", 1049 strerror(ret)); 1050 break; 1051 } 1052 ret = pthread_detach(td->verify_threads[i]); 1053 if (ret) { 1054 log_err("fio: async verify thread detach failed: %s\n", 1055 strerror(ret)); 1056 break; 1057 } 1058 td->nr_verify_threads++; 1059 } 1060 1061 pthread_attr_destroy(&attr); 1062 1063 if (i != td->o.verify_async) { 1064 log_err("fio: only %d verify threads started, exiting\n", i); 1065 td->verify_thread_exit = 1; 1066 write_barrier(); 1067 pthread_cond_broadcast(&td->verify_cond); 1068 return 1; 1069 } 1070 1071 return 0; 1072} 1073 1074void verify_async_exit(struct thread_data *td) 1075{ 1076 td->verify_thread_exit = 1; 1077 write_barrier(); 1078 pthread_cond_broadcast(&td->verify_cond); 1079 1080 pthread_mutex_lock(&td->io_u_lock); 1081 1082 while (td->nr_verify_threads) 1083 pthread_cond_wait(&td->free_cond, &td->io_u_lock); 1084 1085 pthread_mutex_unlock(&td->io_u_lock); 1086 free(td->verify_threads); 1087 td->verify_threads = NULL; 1088} 1089