verify.c revision 304a47c7d94f407cc72a87025679a67f02288447
1/* 2 * IO verification helpers 3 */ 4#include <unistd.h> 5#include <fcntl.h> 6#include <string.h> 7#include <assert.h> 8#include <pthread.h> 9 10#include "fio.h" 11#include "verify.h" 12#include "smalloc.h" 13#include "lib/rand.h" 14 15#include "crc/md5.h" 16#include "crc/crc64.h" 17#include "crc/crc32.h" 18#include "crc/crc32c.h" 19#include "crc/crc16.h" 20#include "crc/crc7.h" 21#include "crc/sha256.h" 22#include "crc/sha512.h" 23#include "crc/sha1.h" 24 25void fill_pattern(struct thread_data *td, void *p, unsigned int len, struct io_u *io_u) 26{ 27 switch (td->o.verify_pattern_bytes) { 28 case 0: 29 dprint(FD_VERIFY, "fill random bytes len=%u\n", len); 30 fill_random_buf(p, len); 31 break; 32 case 1: 33 if (io_u->buf_filled_len >= len) { 34 dprint(FD_VERIFY, "using already filled verify pattern b=0 len=%u\n", len); 35 return; 36 } 37 dprint(FD_VERIFY, "fill verify pattern b=0 len=%u\n", len); 38 memset(p, td->o.verify_pattern[0], len); 39 io_u->buf_filled_len = len; 40 break; 41 default: { 42 unsigned int i = 0, size = 0; 43 unsigned char *b = p; 44 45 if (io_u->buf_filled_len >= len) { 46 dprint(FD_VERIFY, "using already filled verify pattern b=%d len=%u\n", 47 td->o.verify_pattern_bytes, len); 48 return; 49 } 50 dprint(FD_VERIFY, "fill verify pattern b=%d len=%u\n", 51 td->o.verify_pattern_bytes, len); 52 53 while (i < len) { 54 size = td->o.verify_pattern_bytes; 55 if (size > (len - i)) 56 size = len - i; 57 memcpy(b+i, td->o.verify_pattern, size); 58 i += size; 59 } 60 io_u->buf_filled_len = len; 61 break; 62 } 63 } 64} 65 66static void memswp(void *buf1, void *buf2, unsigned int len) 67{ 68 char swap[200]; 69 70 assert(len <= sizeof(swap)); 71 72 memcpy(&swap, buf1, len); 73 memcpy(buf1, buf2, len); 74 memcpy(buf2, &swap, len); 75} 76 77static void hexdump(void *buffer, int len) 78{ 79 unsigned char *p = buffer; 80 int i; 81 82 for (i = 0; i < len; i++) 83 log_err("%02x", p[i]); 84 log_err("\n"); 85} 86 87/* 88 * Prepare for seperation of verify_header and checksum header 89 */ 90static inline unsigned int __hdr_size(int verify_type) 91{ 92 unsigned int len = len; 93 94 switch (verify_type) { 95 case VERIFY_NONE: 96 case VERIFY_NULL: 97 len = 0; 98 break; 99 case VERIFY_MD5: 100 len = sizeof(struct vhdr_md5); 101 break; 102 case VERIFY_CRC64: 103 len = sizeof(struct vhdr_crc64); 104 break; 105 case VERIFY_CRC32C: 106 case VERIFY_CRC32: 107 case VERIFY_CRC32C_INTEL: 108 len = sizeof(struct vhdr_crc32); 109 break; 110 case VERIFY_CRC16: 111 len = sizeof(struct vhdr_crc16); 112 break; 113 case VERIFY_CRC7: 114 len = sizeof(struct vhdr_crc7); 115 break; 116 case VERIFY_SHA256: 117 len = sizeof(struct vhdr_sha256); 118 break; 119 case VERIFY_SHA512: 120 len = sizeof(struct vhdr_sha512); 121 break; 122 case VERIFY_META: 123 len = sizeof(struct vhdr_meta); 124 break; 125 case VERIFY_SHA1: 126 len = sizeof(struct vhdr_sha1); 127 break; 128 default: 129 log_err("fio: unknown verify header!\n"); 130 assert(0); 131 } 132 133 return len + sizeof(struct verify_header); 134} 135 136static inline unsigned int hdr_size(struct verify_header *hdr) 137{ 138 return __hdr_size(hdr->verify_type); 139} 140 141static void *hdr_priv(struct verify_header *hdr) 142{ 143 void *priv = hdr; 144 145 return priv + sizeof(struct verify_header); 146} 147 148/* 149 * Verify container, pass info to verify handlers and allow them to 150 * pass info back in case of error 151 */ 152struct vcont { 153 /* 154 * Input 155 */ 156 struct io_u *io_u; 157 unsigned int hdr_num; 158 159 /* 160 * Output, only valid in case of error 161 */ 162 const char *name; 163 void *good_crc; 164 void *bad_crc; 165 unsigned int crc_len; 166}; 167 168static void log_verify_failure(struct verify_header *hdr, struct vcont *vc) 169{ 170 unsigned long long offset; 171 172 offset = vc->io_u->offset; 173 offset += vc->hdr_num * hdr->len; 174 log_err("%.8s: verify failed at file %s offset %llu, length %u\n", 175 vc->name, vc->io_u->file->file_name, offset, hdr->len); 176 177 if (vc->good_crc && vc->bad_crc) { 178 log_err(" Expected CRC: "); 179 hexdump(vc->good_crc, vc->crc_len); 180 log_err(" Received CRC: "); 181 hexdump(vc->bad_crc, vc->crc_len); 182 } 183} 184 185/* 186 * Return data area 'header_num' 187 */ 188static inline void *io_u_verify_off(struct verify_header *hdr, struct vcont *vc) 189{ 190 return vc->io_u->buf + vc->hdr_num * hdr->len + hdr_size(hdr); 191} 192 193static int verify_io_u_meta(struct verify_header *hdr, struct thread_data *td, 194 struct vcont *vc) 195{ 196 struct vhdr_meta *vh = hdr_priv(hdr); 197 struct io_u *io_u = vc->io_u; 198 199 dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len); 200 201 if (vh->offset == io_u->offset + vc->hdr_num * td->o.verify_interval) 202 return 0; 203 204 vc->name = "meta"; 205 log_verify_failure(hdr, vc); 206 return EILSEQ; 207} 208 209static int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc) 210{ 211 void *p = io_u_verify_off(hdr, vc); 212 struct vhdr_sha512 *vh = hdr_priv(hdr); 213 uint8_t sha512[128]; 214 struct sha512_ctx sha512_ctx = { 215 .buf = sha512, 216 }; 217 218 dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", vc->io_u, hdr->len); 219 220 sha512_init(&sha512_ctx); 221 sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr)); 222 223 if (!memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512))) 224 return 0; 225 226 vc->name = "sha512"; 227 vc->good_crc = vh->sha512; 228 vc->bad_crc = sha512_ctx.buf; 229 vc->crc_len = sizeof(vh->sha512); 230 log_verify_failure(hdr, vc); 231 return EILSEQ; 232} 233 234static int verify_io_u_sha256(struct verify_header *hdr, struct vcont *vc) 235{ 236 void *p = io_u_verify_off(hdr, vc); 237 struct vhdr_sha256 *vh = hdr_priv(hdr); 238 uint8_t sha256[64]; 239 struct sha256_ctx sha256_ctx = { 240 .buf = sha256, 241 }; 242 243 dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", vc->io_u, hdr->len); 244 245 sha256_init(&sha256_ctx); 246 sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr)); 247 248 if (!memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256))) 249 return 0; 250 251 vc->name = "sha256"; 252 vc->good_crc = vh->sha256; 253 vc->bad_crc = sha256_ctx.buf; 254 vc->crc_len = sizeof(vh->sha256); 255 log_verify_failure(hdr, vc); 256 return EILSEQ; 257} 258 259static int verify_io_u_sha1(struct verify_header *hdr, struct vcont *vc) 260{ 261 void *p = io_u_verify_off(hdr, vc); 262 struct vhdr_sha1 *vh = hdr_priv(hdr); 263 uint32_t sha1[5]; 264 struct sha1_ctx sha1_ctx = { 265 .H = sha1, 266 }; 267 268 dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", vc->io_u, hdr->len); 269 270 sha1_init(&sha1_ctx); 271 sha1_update(&sha1_ctx, p, hdr->len - hdr_size(hdr)); 272 273 if (!memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1))) 274 return 0; 275 276 vc->name = "sha1"; 277 vc->good_crc = vh->sha1; 278 vc->bad_crc = sha1_ctx.H; 279 vc->crc_len = sizeof(vh->sha1); 280 log_verify_failure(hdr, vc); 281 return EILSEQ; 282} 283 284static int verify_io_u_crc7(struct verify_header *hdr, struct vcont *vc) 285{ 286 void *p = io_u_verify_off(hdr, vc); 287 struct vhdr_crc7 *vh = hdr_priv(hdr); 288 unsigned char c; 289 290 dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", vc->io_u, hdr->len); 291 292 c = crc7(p, hdr->len - hdr_size(hdr)); 293 294 if (c == vh->crc7) 295 return 0; 296 297 vc->name = "crc7"; 298 vc->good_crc = &vh->crc7; 299 vc->bad_crc = &c; 300 vc->crc_len = 1; 301 log_verify_failure(hdr, vc); 302 return EILSEQ; 303} 304 305static int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc) 306{ 307 void *p = io_u_verify_off(hdr, vc); 308 struct vhdr_crc16 *vh = hdr_priv(hdr); 309 unsigned short c; 310 311 dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len); 312 313 c = crc16(p, hdr->len - hdr_size(hdr)); 314 315 if (c == vh->crc16) 316 return 0; 317 318 vc->name = "crc16"; 319 vc->good_crc = &vh->crc16; 320 vc->bad_crc = &c; 321 vc->crc_len = 2; 322 log_verify_failure(hdr, vc); 323 return EILSEQ; 324} 325 326static int verify_io_u_crc64(struct verify_header *hdr, struct vcont *vc) 327{ 328 void *p = io_u_verify_off(hdr, vc); 329 struct vhdr_crc64 *vh = hdr_priv(hdr); 330 unsigned long long c; 331 332 dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", vc->io_u, hdr->len); 333 334 c = crc64(p, hdr->len - hdr_size(hdr)); 335 336 if (c == vh->crc64) 337 return 0; 338 339 vc->name = "crc64"; 340 vc->good_crc = &vh->crc64; 341 vc->bad_crc = &c; 342 vc->crc_len = 8; 343 log_verify_failure(hdr, vc); 344 return EILSEQ; 345} 346 347static int verify_io_u_crc32(struct verify_header *hdr, struct vcont *vc) 348{ 349 void *p = io_u_verify_off(hdr, vc); 350 struct vhdr_crc32 *vh = hdr_priv(hdr); 351 uint32_t c; 352 353 dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", vc->io_u, hdr->len); 354 355 c = crc32(p, hdr->len - hdr_size(hdr)); 356 357 if (c == vh->crc32) 358 return 0; 359 360 vc->name = "crc32"; 361 vc->good_crc = &vh->crc32; 362 vc->bad_crc = &c; 363 vc->crc_len = 4; 364 log_verify_failure(hdr, vc); 365 return EILSEQ; 366} 367 368static int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc) 369{ 370 void *p = io_u_verify_off(hdr, vc); 371 struct vhdr_crc32 *vh = hdr_priv(hdr); 372 uint32_t c; 373 374 dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len); 375 376 if (hdr->verify_type == VERIFY_CRC32C_INTEL) 377 c = crc32c_intel(p, hdr->len - hdr_size(hdr)); 378 else 379 c = crc32c(p, hdr->len - hdr_size(hdr)); 380 381 if (c == vh->crc32) 382 return 0; 383 384 vc->name = "crc32c"; 385 vc->good_crc = &vh->crc32; 386 vc->bad_crc = &c; 387 vc->crc_len = 4; 388 log_verify_failure(hdr, vc); 389 return EILSEQ; 390} 391 392static int verify_io_u_md5(struct verify_header *hdr, struct vcont *vc) 393{ 394 void *p = io_u_verify_off(hdr, vc); 395 struct vhdr_md5 *vh = hdr_priv(hdr); 396 uint32_t hash[MD5_HASH_WORDS]; 397 struct md5_ctx md5_ctx = { 398 .hash = hash, 399 }; 400 401 dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", vc->io_u, hdr->len); 402 403 md5_init(&md5_ctx); 404 md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr)); 405 406 if (!memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash))) 407 return 0; 408 409 vc->name = "md5"; 410 vc->good_crc = vh->md5_digest; 411 vc->bad_crc = md5_ctx.hash; 412 vc->crc_len = sizeof(hash); 413 log_verify_failure(hdr, vc); 414 return EILSEQ; 415} 416 417static unsigned int hweight8(unsigned int w) 418{ 419 unsigned int res = w - ((w >> 1) & 0x55); 420 421 res = (res & 0x33) + ((res >> 2) & 0x33); 422 return (res + (res >> 4)) & 0x0F; 423} 424 425int verify_io_u_pattern(char *pattern, unsigned long pattern_size, 426 char *buf, unsigned int len, unsigned int mod) 427{ 428 unsigned int i; 429 430 for (i = 0; i < len; i++) { 431 if (buf[i] != pattern[mod]) { 432 unsigned int bits; 433 434 bits = hweight8(buf[i] ^ pattern[mod]); 435 log_err("fio: got pattern %x, wanted %x. Bad bits %d\n", 436 buf[i], pattern[mod], bits); 437 log_err("fio: bad pattern block offset %u\n", i); 438 return EILSEQ; 439 } 440 mod++; 441 if (mod == pattern_size) 442 mod = 0; 443 } 444 445 return 0; 446} 447 448/* 449 * Push IO verification to a separate thread 450 */ 451int verify_io_u_async(struct thread_data *td, struct io_u *io_u) 452{ 453 if (io_u->file) 454 put_file_log(td, io_u->file); 455 456 io_u->file = NULL; 457 458 pthread_mutex_lock(&td->io_u_lock); 459 460 if (io_u->flags & IO_U_F_IN_CUR_DEPTH) { 461 td->cur_depth--; 462 io_u->flags &= ~IO_U_F_IN_CUR_DEPTH; 463 } 464 flist_del(&io_u->list); 465 flist_add_tail(&io_u->list, &td->verify_list); 466 io_u->flags |= IO_U_F_FREE_DEF; 467 pthread_mutex_unlock(&td->io_u_lock); 468 469 pthread_cond_signal(&td->verify_cond); 470 return 0; 471} 472 473int verify_io_u(struct thread_data *td, struct io_u *io_u) 474{ 475 struct verify_header *hdr; 476 unsigned int hdr_size, hdr_inc, hdr_num = 0; 477 void *p; 478 int ret; 479 480 if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ) 481 return 0; 482 483 hdr_inc = io_u->buflen; 484 if (td->o.verify_interval) 485 hdr_inc = td->o.verify_interval; 486 487 ret = 0; 488 for (p = io_u->buf; p < io_u->buf + io_u->buflen; 489 p += hdr_inc, hdr_num++) { 490 struct vcont vc = { 491 .io_u = io_u, 492 .hdr_num = hdr_num, 493 }; 494 495 if (ret && td->o.verify_fatal) 496 break; 497 498 hdr_size = __hdr_size(td->o.verify); 499 if (td->o.verify_offset) 500 memswp(p, p + td->o.verify_offset, hdr_size); 501 hdr = p; 502 503 if (hdr->fio_magic != FIO_HDR_MAGIC) { 504 log_err("verify: bad magic header %x, wanted %x at file %s offset %llu, length %u\n", 505 hdr->fio_magic, FIO_HDR_MAGIC, 506 io_u->file->file_name, 507 io_u->offset + hdr_num * hdr->len, hdr->len); 508 return EILSEQ; 509 } 510 511 if (td->o.verify_pattern_bytes) { 512 dprint(FD_VERIFY, "pattern verify io_u %p, len %u\n", 513 io_u, hdr->len); 514 ret = verify_io_u_pattern(td->o.verify_pattern, 515 td->o.verify_pattern_bytes, 516 p + hdr_size, 517 hdr_inc - hdr_size, 518 hdr_size % td->o.verify_pattern_bytes); 519 520 if (ret) { 521 log_err("pattern: verify failed at file %s offset %llu, length %u\n", 522 io_u->file->file_name, 523 io_u->offset + hdr_num * hdr->len, 524 hdr->len); 525 } 526 527 /* 528 * Also verify the meta data, if applicable 529 */ 530 if (hdr->verify_type == VERIFY_META) 531 ret |= verify_io_u_meta(hdr, td, &vc); 532 continue; 533 } 534 535 switch (hdr->verify_type) { 536 case VERIFY_MD5: 537 ret = verify_io_u_md5(hdr, &vc); 538 break; 539 case VERIFY_CRC64: 540 ret = verify_io_u_crc64(hdr, &vc); 541 break; 542 case VERIFY_CRC32C: 543 case VERIFY_CRC32C_INTEL: 544 ret = verify_io_u_crc32c(hdr, &vc); 545 break; 546 case VERIFY_CRC32: 547 ret = verify_io_u_crc32(hdr, &vc); 548 break; 549 case VERIFY_CRC16: 550 ret = verify_io_u_crc16(hdr, &vc); 551 break; 552 case VERIFY_CRC7: 553 ret = verify_io_u_crc7(hdr, &vc); 554 break; 555 case VERIFY_SHA256: 556 ret = verify_io_u_sha256(hdr, &vc); 557 break; 558 case VERIFY_SHA512: 559 ret = verify_io_u_sha512(hdr, &vc); 560 break; 561 case VERIFY_META: 562 ret = verify_io_u_meta(hdr, td, &vc); 563 break; 564 case VERIFY_SHA1: 565 ret = verify_io_u_sha1(hdr, &vc); 566 break; 567 default: 568 log_err("Bad verify type %u\n", hdr->verify_type); 569 ret = EINVAL; 570 } 571 } 572 573 if (ret && td->o.verify_fatal) 574 td->terminate = 1; 575 576 return ret; 577} 578 579static void fill_meta(struct verify_header *hdr, struct thread_data *td, 580 struct io_u *io_u, unsigned int header_num) 581{ 582 struct vhdr_meta *vh = hdr_priv(hdr); 583 584 vh->thread = td->thread_number; 585 586 vh->time_sec = io_u->start_time.tv_sec; 587 vh->time_usec = io_u->start_time.tv_usec; 588 589 vh->numberio = td->io_issues[DDIR_WRITE]; 590 591 vh->offset = io_u->offset + header_num * td->o.verify_interval; 592} 593 594static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len) 595{ 596 struct vhdr_sha512 *vh = hdr_priv(hdr); 597 struct sha512_ctx sha512_ctx = { 598 .buf = vh->sha512, 599 }; 600 601 sha512_init(&sha512_ctx); 602 sha512_update(&sha512_ctx, p, len); 603} 604 605static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len) 606{ 607 struct vhdr_sha256 *vh = hdr_priv(hdr); 608 struct sha256_ctx sha256_ctx = { 609 .buf = vh->sha256, 610 }; 611 612 sha256_init(&sha256_ctx); 613 sha256_update(&sha256_ctx, p, len); 614} 615 616static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len) 617{ 618 struct vhdr_sha1 *vh = hdr_priv(hdr); 619 struct sha1_ctx sha1_ctx = { 620 .H = vh->sha1, 621 }; 622 623 sha1_init(&sha1_ctx); 624 sha1_update(&sha1_ctx, p, len); 625} 626 627static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len) 628{ 629 struct vhdr_crc7 *vh = hdr_priv(hdr); 630 631 vh->crc7 = crc7(p, len); 632} 633 634static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len) 635{ 636 struct vhdr_crc16 *vh = hdr_priv(hdr); 637 638 vh->crc16 = crc16(p, len); 639} 640 641static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len) 642{ 643 struct vhdr_crc32 *vh = hdr_priv(hdr); 644 645 vh->crc32 = crc32(p, len); 646} 647 648static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len) 649{ 650 struct vhdr_crc32 *vh = hdr_priv(hdr); 651 652 if (hdr->verify_type == VERIFY_CRC32C_INTEL) 653 vh->crc32 = crc32c_intel(p, len); 654 else 655 vh->crc32 = crc32c(p, len); 656} 657 658static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len) 659{ 660 struct vhdr_crc64 *vh = hdr_priv(hdr); 661 662 vh->crc64 = crc64(p, len); 663} 664 665static void fill_md5(struct verify_header *hdr, void *p, unsigned int len) 666{ 667 struct vhdr_md5 *vh = hdr_priv(hdr); 668 struct md5_ctx md5_ctx = { 669 .hash = (uint32_t *) vh->md5_digest, 670 }; 671 672 md5_init(&md5_ctx); 673 md5_update(&md5_ctx, p, len); 674} 675 676/* 677 * fill body of io_u->buf with random data and add a header with the 678 * crc32 or md5 sum of that data. 679 */ 680void populate_verify_io_u(struct thread_data *td, struct io_u *io_u) 681{ 682 struct verify_header *hdr; 683 void *p = io_u->buf, *data; 684 unsigned int hdr_inc, data_len, header_num = 0; 685 686 if (td->o.verify == VERIFY_NULL) 687 return; 688 689 fill_pattern(td, p, io_u->buflen, io_u); 690 691 hdr_inc = io_u->buflen; 692 if (td->o.verify_interval) 693 hdr_inc = td->o.verify_interval; 694 695 for (; p < io_u->buf + io_u->buflen; p += hdr_inc) { 696 hdr = p; 697 698 hdr->fio_magic = FIO_HDR_MAGIC; 699 hdr->verify_type = td->o.verify; 700 hdr->len = hdr_inc; 701 data_len = hdr_inc - hdr_size(hdr); 702 703 data = p + hdr_size(hdr); 704 switch (td->o.verify) { 705 case VERIFY_MD5: 706 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n", 707 io_u, hdr->len); 708 fill_md5(hdr, data, data_len); 709 break; 710 case VERIFY_CRC64: 711 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n", 712 io_u, hdr->len); 713 fill_crc64(hdr, data, data_len); 714 break; 715 case VERIFY_CRC32C: 716 case VERIFY_CRC32C_INTEL: 717 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n", 718 io_u, hdr->len); 719 fill_crc32c(hdr, data, data_len); 720 break; 721 case VERIFY_CRC32: 722 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n", 723 io_u, hdr->len); 724 fill_crc32(hdr, data, data_len); 725 break; 726 case VERIFY_CRC16: 727 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n", 728 io_u, hdr->len); 729 fill_crc16(hdr, data, data_len); 730 break; 731 case VERIFY_CRC7: 732 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n", 733 io_u, hdr->len); 734 fill_crc7(hdr, data, data_len); 735 break; 736 case VERIFY_SHA256: 737 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n", 738 io_u, hdr->len); 739 fill_sha256(hdr, data, data_len); 740 break; 741 case VERIFY_SHA512: 742 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n", 743 io_u, hdr->len); 744 fill_sha512(hdr, data, data_len); 745 break; 746 case VERIFY_META: 747 dprint(FD_VERIFY, "fill meta io_u %p, len %u\n", 748 io_u, hdr->len); 749 fill_meta(hdr, td, io_u, header_num); 750 break; 751 case VERIFY_SHA1: 752 dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n", 753 io_u, hdr->len); 754 fill_sha1(hdr, data, data_len); 755 break; 756 default: 757 log_err("fio: bad verify type: %d\n", td->o.verify); 758 assert(0); 759 } 760 if (td->o.verify_offset) 761 memswp(p, p + td->o.verify_offset, hdr_size(hdr)); 762 header_num++; 763 } 764} 765 766int get_next_verify(struct thread_data *td, struct io_u *io_u) 767{ 768 struct io_piece *ipo = NULL; 769 770 /* 771 * this io_u is from a requeue, we already filled the offsets 772 */ 773 if (io_u->file) 774 return 0; 775 776 if (!RB_EMPTY_ROOT(&td->io_hist_tree)) { 777 struct rb_node *n = rb_first(&td->io_hist_tree); 778 779 ipo = rb_entry(n, struct io_piece, rb_node); 780 rb_erase(n, &td->io_hist_tree); 781 td->io_hist_len--; 782 } else if (!flist_empty(&td->io_hist_list)) { 783 ipo = flist_entry(td->io_hist_list.next, struct io_piece, list); 784 td->io_hist_len--; 785 flist_del(&ipo->list); 786 } 787 788 if (ipo) { 789 io_u->offset = ipo->offset; 790 io_u->buflen = ipo->len; 791 io_u->file = ipo->file; 792 793 if (!fio_file_open(io_u->file)) { 794 int r = td_io_open_file(td, io_u->file); 795 796 if (r) { 797 dprint(FD_VERIFY, "failed file %s open\n", 798 io_u->file->file_name); 799 return 1; 800 } 801 } 802 803 get_file(ipo->file); 804 assert(fio_file_open(io_u->file)); 805 io_u->ddir = DDIR_READ; 806 io_u->xfer_buf = io_u->buf; 807 io_u->xfer_buflen = io_u->buflen; 808 free(ipo); 809 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u); 810 return 0; 811 } 812 813 dprint(FD_VERIFY, "get_next_verify: empty\n"); 814 return 1; 815} 816 817static void *verify_async_thread(void *data) 818{ 819 struct thread_data *td = data; 820 struct io_u *io_u; 821 int ret = 0; 822 823 if (td->o.verify_cpumask_set && 824 fio_setaffinity(td->pid, td->o.verify_cpumask)) { 825 log_err("fio: failed setting verify thread affinity\n"); 826 goto done; 827 } 828 829 do { 830 FLIST_HEAD(list); 831 832 read_barrier(); 833 if (td->verify_thread_exit) 834 break; 835 836 pthread_mutex_lock(&td->io_u_lock); 837 838 while (flist_empty(&td->verify_list) && 839 !td->verify_thread_exit) { 840 ret = pthread_cond_wait(&td->verify_cond, 841 &td->io_u_lock); 842 if (ret) { 843 pthread_mutex_unlock(&td->io_u_lock); 844 break; 845 } 846 } 847 848 flist_splice_init(&td->verify_list, &list); 849 pthread_mutex_unlock(&td->io_u_lock); 850 851 if (flist_empty(&list)) 852 continue; 853 854 while (!flist_empty(&list)) { 855 io_u = flist_entry(list.next, struct io_u, list); 856 flist_del_init(&io_u->list); 857 858 ret = verify_io_u(td, io_u); 859 put_io_u(td, io_u); 860 if (!ret) 861 continue; 862 if (td->o.continue_on_error && 863 td_non_fatal_error(ret)) { 864 update_error_count(td, ret); 865 td_clear_error(td); 866 ret = 0; 867 } 868 } 869 } while (!ret); 870 871 if (ret) { 872 td_verror(td, ret, "async_verify"); 873 if (td->o.verify_fatal) 874 td->terminate = 1; 875 } 876 877done: 878 pthread_mutex_lock(&td->io_u_lock); 879 td->nr_verify_threads--; 880 pthread_mutex_unlock(&td->io_u_lock); 881 882 pthread_cond_signal(&td->free_cond); 883 return NULL; 884} 885 886int verify_async_init(struct thread_data *td) 887{ 888 int i, ret; 889 pthread_attr_t attr; 890 891 pthread_attr_init(&attr); 892 pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN); 893 894 td->verify_thread_exit = 0; 895 896 td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async); 897 for (i = 0; i < td->o.verify_async; i++) { 898 ret = pthread_create(&td->verify_threads[i], &attr, 899 verify_async_thread, td); 900 if (ret) { 901 log_err("fio: async verify creation failed: %s\n", 902 strerror(ret)); 903 break; 904 } 905 ret = pthread_detach(td->verify_threads[i]); 906 if (ret) { 907 log_err("fio: async verify thread detach failed: %s\n", 908 strerror(ret)); 909 break; 910 } 911 td->nr_verify_threads++; 912 } 913 914 pthread_attr_destroy(&attr); 915 916 if (i != td->o.verify_async) { 917 log_err("fio: only %d verify threads started, exiting\n", i); 918 td->verify_thread_exit = 1; 919 write_barrier(); 920 pthread_cond_broadcast(&td->verify_cond); 921 return 1; 922 } 923 924 return 0; 925} 926 927void verify_async_exit(struct thread_data *td) 928{ 929 td->verify_thread_exit = 1; 930 write_barrier(); 931 pthread_cond_broadcast(&td->verify_cond); 932 933 pthread_mutex_lock(&td->io_u_lock); 934 935 while (td->nr_verify_threads) 936 pthread_cond_wait(&td->free_cond, &td->io_u_lock); 937 938 pthread_mutex_unlock(&td->io_u_lock); 939 free(td->verify_threads); 940 td->verify_threads = NULL; 941} 942