verify.c revision b6bf4995922789978a43954c53d501b159d15b0e
1/* 2 * IO verification helpers 3 */ 4#include <unistd.h> 5#include <fcntl.h> 6#include <string.h> 7#include <assert.h> 8#include <pthread.h> 9 10#include "fio.h" 11#include "verify.h" 12#include "smalloc.h" 13 14#include "crc/md5.h" 15#include "crc/crc64.h" 16#include "crc/crc32.h" 17#include "crc/crc32c.h" 18#include "crc/crc16.h" 19#include "crc/crc7.h" 20#include "crc/sha256.h" 21#include "crc/sha512.h" 22#include "crc/sha1.h" 23 24static void fill_random_bytes(struct thread_data *td, void *p, unsigned int len) 25{ 26 unsigned int todo; 27 int r; 28 29 while (len) { 30 r = os_random_long(&td->verify_state); 31 32 /* 33 * lrand48_r seems to be broken and only fill the bottom 34 * 32-bits, even on 64-bit archs with 64-bit longs 35 */ 36 todo = sizeof(r); 37 if (todo > len) 38 todo = len; 39 40 memcpy(p, &r, todo); 41 42 len -= todo; 43 p += todo; 44 } 45} 46 47static void fill_pattern(struct thread_data *td, void *p, unsigned int len) 48{ 49 switch (td->o.verify_pattern_bytes) { 50 case 0: 51 dprint(FD_VERIFY, "fill random bytes len=%u\n", len); 52 fill_random_bytes(td, p, len); 53 break; 54 case 1: 55 dprint(FD_VERIFY, "fill verify pattern b=0 len=%u\n", len); 56 memset(p, td->o.verify_pattern[0], len); 57 break; 58 default: { 59 unsigned int i = 0, size = 0; 60 unsigned char *b = p; 61 62 dprint(FD_VERIFY, "fill verify pattern b=%d len=%u\n", 63 td->o.verify_pattern_bytes, len); 64 65 while (i < len) { 66 size = td->o.verify_pattern_bytes; 67 if (size > (len - i)) 68 size = len - i; 69 memcpy(b+i, td->o.verify_pattern, size); 70 i += size; 71 } 72 break; 73 } 74 } 75} 76 77static void memswp(void *buf1, void *buf2, unsigned int len) 78{ 79 char swap[200]; 80 81 assert(len <= sizeof(swap)); 82 83 memcpy(&swap, buf1, len); 84 memcpy(buf1, buf2, len); 85 memcpy(buf2, &swap, len); 86} 87 88static void hexdump(void *buffer, int len) 89{ 90 unsigned char *p = buffer; 91 int i; 92 93 for (i = 0; i < len; i++) 94 log_info("%02x", p[i]); 95 log_info("\n"); 96} 97 98/* 99 * Prepare for seperation of verify_header and checksum header 100 */ 101static inline unsigned int __hdr_size(int verify_type) 102{ 103 unsigned int len = len; 104 105 switch (verify_type) { 106 case VERIFY_NONE: 107 case VERIFY_NULL: 108 len = 0; 109 break; 110 case VERIFY_MD5: 111 len = sizeof(struct vhdr_md5); 112 break; 113 case VERIFY_CRC64: 114 len = sizeof(struct vhdr_crc64); 115 break; 116 case VERIFY_CRC32C: 117 case VERIFY_CRC32: 118 case VERIFY_CRC32C_INTEL: 119 len = sizeof(struct vhdr_crc32); 120 break; 121 case VERIFY_CRC16: 122 len = sizeof(struct vhdr_crc16); 123 break; 124 case VERIFY_CRC7: 125 len = sizeof(struct vhdr_crc7); 126 break; 127 case VERIFY_SHA256: 128 len = sizeof(struct vhdr_sha256); 129 break; 130 case VERIFY_SHA512: 131 len = sizeof(struct vhdr_sha512); 132 break; 133 case VERIFY_META: 134 len = sizeof(struct vhdr_meta); 135 break; 136 case VERIFY_SHA1: 137 len = sizeof(struct vhdr_sha1); 138 break; 139 default: 140 log_err("fio: unknown verify header!\n"); 141 assert(0); 142 } 143 144 return len + sizeof(struct verify_header); 145} 146 147static inline unsigned int hdr_size(struct verify_header *hdr) 148{ 149 return __hdr_size(hdr->verify_type); 150} 151 152static void *hdr_priv(struct verify_header *hdr) 153{ 154 void *priv = hdr; 155 156 return priv + sizeof(struct verify_header); 157} 158 159/* 160 * Return data area 'header_num' 161 */ 162static inline void *io_u_verify_off(struct verify_header *hdr, 163 struct io_u *io_u, unsigned int header_num) 164{ 165 return io_u->buf + header_num * hdr->len + hdr_size(hdr); 166} 167 168static int verify_io_u_meta(struct verify_header *hdr, struct thread_data *td, 169 struct io_u *io_u, unsigned int header_num) 170{ 171 struct vhdr_meta *vh = hdr_priv(hdr); 172 173 dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len); 174 175 if (vh->offset != io_u->offset + header_num * td->o.verify_interval) { 176 log_err("meta: verify failed at %llu/%u\n", 177 io_u->offset + header_num * hdr->len, hdr->len); 178 return EILSEQ; 179 } 180 181 return 0; 182} 183 184static int verify_io_u_sha512(struct verify_header *hdr, struct io_u *io_u, 185 unsigned int header_num) 186{ 187 void *p = io_u_verify_off(hdr, io_u, header_num); 188 struct vhdr_sha512 *vh = hdr_priv(hdr); 189 uint8_t sha512[128]; 190 struct sha512_ctx sha512_ctx = { 191 .buf = sha512, 192 }; 193 194 dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", io_u, hdr->len); 195 196 sha512_init(&sha512_ctx); 197 sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr)); 198 199 if (memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512))) { 200 log_err("sha512: verify failed at %llu/%u\n", 201 io_u->offset + header_num * hdr->len, hdr->len); 202 hexdump(vh->sha512, sizeof(vh->sha512)); 203 hexdump(sha512_ctx.buf, sizeof(sha512)); 204 return EILSEQ; 205 } 206 207 return 0; 208} 209 210static int verify_io_u_sha256(struct verify_header *hdr, struct io_u *io_u, 211 unsigned int header_num) 212{ 213 void *p = io_u_verify_off(hdr, io_u, header_num); 214 struct vhdr_sha256 *vh = hdr_priv(hdr); 215 uint8_t sha256[64]; 216 struct sha256_ctx sha256_ctx = { 217 .buf = sha256, 218 }; 219 220 dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", io_u, hdr->len); 221 222 sha256_init(&sha256_ctx); 223 sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr)); 224 225 if (memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256))) { 226 log_err("sha256: verify failed at %llu/%u\n", 227 io_u->offset + header_num * hdr->len, hdr->len); 228 hexdump(vh->sha256, sizeof(vh->sha256)); 229 hexdump(sha256_ctx.buf, sizeof(sha256)); 230 return EILSEQ; 231 } 232 233 return 0; 234} 235 236static int verify_io_u_sha1(struct verify_header *hdr, struct io_u *io_u, 237 unsigned int header_num) 238{ 239 void *p = io_u_verify_off(hdr, io_u, header_num); 240 struct vhdr_sha1 *vh = hdr_priv(hdr); 241 uint32_t sha1[5]; 242 struct sha1_ctx sha1_ctx = { 243 .H = sha1, 244 }; 245 246 dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", io_u, hdr->len); 247 248 sha1_init(&sha1_ctx); 249 sha1_update(&sha1_ctx, p, hdr->len - hdr_size(hdr)); 250 251 if (memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1))) { 252 log_err("sha1: verify failed at %llu/%u\n", 253 io_u->offset + header_num * hdr->len, hdr->len); 254 hexdump(vh->sha1, sizeof(vh->sha1)); 255 hexdump(sha1_ctx.H, sizeof(sha1)); 256 return EILSEQ; 257 } 258 259 return 0; 260} 261 262static int verify_io_u_crc7(struct verify_header *hdr, struct io_u *io_u, 263 unsigned int header_num) 264{ 265 void *p = io_u_verify_off(hdr, io_u, header_num); 266 struct vhdr_crc7 *vh = hdr_priv(hdr); 267 unsigned char c; 268 269 dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", io_u, hdr->len); 270 271 c = crc7(p, hdr->len - hdr_size(hdr)); 272 273 if (c != vh->crc7) { 274 log_err("crc7: verify failed at %llu/%u\n", 275 io_u->offset + header_num * hdr->len, hdr->len); 276 log_err("crc7: wanted %x, got %x\n", vh->crc7, c); 277 return EILSEQ; 278 } 279 280 return 0; 281} 282 283static int verify_io_u_crc16(struct verify_header *hdr, struct io_u *io_u, 284 unsigned int header_num) 285{ 286 void *p = io_u_verify_off(hdr, io_u, header_num); 287 struct vhdr_crc16 *vh = hdr_priv(hdr); 288 unsigned short c; 289 290 dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", io_u, hdr->len); 291 292 c = crc16(p, hdr->len - hdr_size(hdr)); 293 294 if (c != vh->crc16) { 295 log_err("crc16: verify failed at %llu/%u\n", 296 io_u->offset + header_num * hdr->len, hdr->len); 297 log_err("crc16: wanted %x, got %x\n", vh->crc16, c); 298 return EILSEQ; 299 } 300 301 return 0; 302} 303 304static int verify_io_u_crc64(struct verify_header *hdr, struct io_u *io_u, 305 unsigned int header_num) 306{ 307 void *p = io_u_verify_off(hdr, io_u, header_num); 308 struct vhdr_crc64 *vh = hdr_priv(hdr); 309 unsigned long long c; 310 311 dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", io_u, hdr->len); 312 313 c = crc64(p, hdr->len - hdr_size(hdr)); 314 315 if (c != vh->crc64) { 316 log_err("crc64: verify failed at %llu/%u\n", 317 io_u->offset + header_num * hdr->len, 318 hdr->len); 319 log_err("crc64: wanted %llx, got %llx\n", 320 (unsigned long long) vh->crc64, c); 321 return EILSEQ; 322 } 323 324 return 0; 325} 326 327static int verify_io_u_crc32(struct verify_header *hdr, struct io_u *io_u, 328 unsigned int header_num) 329{ 330 void *p = io_u_verify_off(hdr, io_u, header_num); 331 struct vhdr_crc32 *vh = hdr_priv(hdr); 332 uint32_t c; 333 334 dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", io_u, hdr->len); 335 336 c = crc32(p, hdr->len - hdr_size(hdr)); 337 338 if (c != vh->crc32) { 339 log_err("crc32: verify failed at %llu/%u\n", 340 io_u->offset + header_num * hdr->len, hdr->len); 341 log_err("crc32: wanted %x, got %x\n", vh->crc32, c); 342 return EILSEQ; 343 } 344 345 return 0; 346} 347 348static int verify_io_u_crc32c(struct verify_header *hdr, struct io_u *io_u, 349 unsigned int header_num) 350{ 351 void *p = io_u_verify_off(hdr, io_u, header_num); 352 struct vhdr_crc32 *vh = hdr_priv(hdr); 353 uint32_t c; 354 355 dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", io_u, hdr->len); 356 357 if (hdr->verify_type == VERIFY_CRC32C_INTEL) 358 c = crc32c_intel(p, hdr->len - hdr_size(hdr)); 359 else 360 c = crc32c(p, hdr->len - hdr_size(hdr)); 361 362 if (c != vh->crc32) { 363 log_err("crc32c: verify failed at %llu/%u\n", 364 io_u->offset + header_num * hdr->len, hdr->len); 365 log_err("crc32c: wanted %x, got %x\n", vh->crc32, c); 366 return EILSEQ; 367 } 368 369 return 0; 370} 371 372static int verify_io_u_md5(struct verify_header *hdr, struct io_u *io_u, 373 unsigned int header_num) 374{ 375 void *p = io_u_verify_off(hdr, io_u, header_num); 376 struct vhdr_md5 *vh = hdr_priv(hdr); 377 uint32_t hash[MD5_HASH_WORDS]; 378 struct md5_ctx md5_ctx = { 379 .hash = hash, 380 }; 381 382 dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", io_u, hdr->len); 383 384 md5_init(&md5_ctx); 385 md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr)); 386 387 if (memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash))) { 388 log_err("md5: verify failed at %llu/%u\n", 389 io_u->offset + header_num * hdr->len, hdr->len); 390 hexdump(vh->md5_digest, sizeof(vh->md5_digest)); 391 hexdump(md5_ctx.hash, sizeof(hash)); 392 return EILSEQ; 393 } 394 395 return 0; 396} 397 398static unsigned int hweight8(unsigned int w) 399{ 400 unsigned int res = w - ((w >> 1) & 0x55); 401 402 res = (res & 0x33) + ((res >> 2) & 0x33); 403 return (res + (res >> 4)) & 0x0F; 404} 405 406int verify_io_u_pattern(char *pattern, unsigned long pattern_size, 407 char *buf, unsigned int len, unsigned int mod) 408{ 409 unsigned int i; 410 411 for (i = 0; i < len; i++) { 412 if (buf[i] != pattern[mod]) { 413 unsigned int bits; 414 415 bits = hweight8(buf[i] ^ pattern[mod]); 416 log_err("fio: got pattern %x, wanted %x. Bad bits %d\n", 417 buf[i], pattern[mod], bits); 418 log_err("fio: bad pattern block offset %u\n", i); 419 return EILSEQ; 420 } 421 mod++; 422 if (mod == pattern_size) 423 mod = 0; 424 } 425 426 return 0; 427} 428 429/* 430 * Push IO verification to a separate thread 431 */ 432int verify_io_u_async(struct thread_data *td, struct io_u *io_u) 433{ 434 if (io_u->file) 435 put_file_log(td, io_u->file); 436 437 io_u->file = NULL; 438 439 pthread_mutex_lock(&td->io_u_lock); 440 441 if (io_u->flags & IO_U_F_IN_CUR_DEPTH) { 442 td->cur_depth--; 443 io_u->flags &= ~IO_U_F_IN_CUR_DEPTH; 444 } 445 flist_del(&io_u->list); 446 flist_add_tail(&io_u->list, &td->verify_list); 447 io_u->flags |= IO_U_F_FREE_DEF; 448 pthread_mutex_unlock(&td->io_u_lock); 449 450 pthread_cond_signal(&td->verify_cond); 451 return 0; 452} 453 454int verify_io_u(struct thread_data *td, struct io_u *io_u) 455{ 456 struct verify_header *hdr; 457 unsigned int hdr_size, hdr_inc, hdr_num = 0; 458 void *p; 459 int ret; 460 461 if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ) 462 return 0; 463 464 hdr_inc = io_u->buflen; 465 if (td->o.verify_interval) 466 hdr_inc = td->o.verify_interval; 467 468 ret = 0; 469 for (p = io_u->buf; p < io_u->buf + io_u->buflen; 470 p += hdr_inc, hdr_num++) { 471 if (ret && td->o.verify_fatal) { 472 td->terminate = 1; 473 break; 474 } 475 hdr_size = __hdr_size(td->o.verify); 476 if (td->o.verify_offset) 477 memswp(p, p + td->o.verify_offset, hdr_size); 478 hdr = p; 479 480 if (hdr->fio_magic != FIO_HDR_MAGIC) { 481 log_err("Bad verify header %x at %llu\n", 482 hdr->fio_magic, 483 io_u->offset + hdr_num * hdr->len); 484 return EILSEQ; 485 } 486 487 if (td->o.verify_pattern_bytes) { 488 dprint(FD_VERIFY, "pattern verify io_u %p, len %u\n", 489 io_u, hdr->len); 490 ret = verify_io_u_pattern(td->o.verify_pattern, 491 td->o.verify_pattern_bytes, 492 p + hdr_size, 493 hdr_inc - hdr_size, 494 hdr_size % td->o.verify_pattern_bytes); 495 /* 496 * Also verify the meta data, if applicable 497 */ 498 if (hdr->verify_type == VERIFY_META) 499 ret |= verify_io_u_meta(hdr, td, io_u, hdr_num); 500 501 if (ret) 502 log_err("fio: verify failed at %llu/%u\n", 503 io_u->offset + hdr_num * hdr->len, 504 hdr->len); 505 continue; 506 } 507 508 switch (hdr->verify_type) { 509 case VERIFY_MD5: 510 ret = verify_io_u_md5(hdr, io_u, hdr_num); 511 break; 512 case VERIFY_CRC64: 513 ret = verify_io_u_crc64(hdr, io_u, hdr_num); 514 break; 515 case VERIFY_CRC32C: 516 case VERIFY_CRC32C_INTEL: 517 ret = verify_io_u_crc32c(hdr, io_u, hdr_num); 518 break; 519 case VERIFY_CRC32: 520 ret = verify_io_u_crc32(hdr, io_u, hdr_num); 521 break; 522 case VERIFY_CRC16: 523 ret = verify_io_u_crc16(hdr, io_u, hdr_num); 524 break; 525 case VERIFY_CRC7: 526 ret = verify_io_u_crc7(hdr, io_u, hdr_num); 527 break; 528 case VERIFY_SHA256: 529 ret = verify_io_u_sha256(hdr, io_u, hdr_num); 530 break; 531 case VERIFY_SHA512: 532 ret = verify_io_u_sha512(hdr, io_u, hdr_num); 533 break; 534 case VERIFY_META: 535 ret = verify_io_u_meta(hdr, td, io_u, hdr_num); 536 break; 537 case VERIFY_SHA1: 538 ret = verify_io_u_sha1(hdr, io_u, hdr_num); 539 break; 540 default: 541 log_err("Bad verify type %u\n", hdr->verify_type); 542 ret = EINVAL; 543 } 544 } 545 546 return ret; 547} 548 549static void fill_meta(struct verify_header *hdr, struct thread_data *td, 550 struct io_u *io_u, unsigned int header_num) 551{ 552 struct vhdr_meta *vh = hdr_priv(hdr); 553 554 vh->thread = td->thread_number; 555 556 vh->time_sec = io_u->start_time.tv_sec; 557 vh->time_usec = io_u->start_time.tv_usec; 558 559 vh->numberio = td->io_issues[DDIR_WRITE]; 560 561 vh->offset = io_u->offset + header_num * td->o.verify_interval; 562} 563 564static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len) 565{ 566 struct vhdr_sha512 *vh = hdr_priv(hdr); 567 struct sha512_ctx sha512_ctx = { 568 .buf = vh->sha512, 569 }; 570 571 sha512_init(&sha512_ctx); 572 sha512_update(&sha512_ctx, p, len); 573} 574 575static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len) 576{ 577 struct vhdr_sha256 *vh = hdr_priv(hdr); 578 struct sha256_ctx sha256_ctx = { 579 .buf = vh->sha256, 580 }; 581 582 sha256_init(&sha256_ctx); 583 sha256_update(&sha256_ctx, p, len); 584} 585 586static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len) 587{ 588 struct vhdr_sha1 *vh = hdr_priv(hdr); 589 struct sha1_ctx sha1_ctx = { 590 .H = vh->sha1, 591 }; 592 593 sha1_init(&sha1_ctx); 594 sha1_update(&sha1_ctx, p, len); 595} 596 597static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len) 598{ 599 struct vhdr_crc7 *vh = hdr_priv(hdr); 600 601 vh->crc7 = crc7(p, len); 602} 603 604static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len) 605{ 606 struct vhdr_crc16 *vh = hdr_priv(hdr); 607 608 vh->crc16 = crc16(p, len); 609} 610 611static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len) 612{ 613 struct vhdr_crc32 *vh = hdr_priv(hdr); 614 615 vh->crc32 = crc32(p, len); 616} 617 618static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len) 619{ 620 struct vhdr_crc32 *vh = hdr_priv(hdr); 621 622 if (hdr->verify_type == VERIFY_CRC32C_INTEL) 623 vh->crc32 = crc32c_intel(p, len); 624 else 625 vh->crc32 = crc32c(p, len); 626} 627 628static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len) 629{ 630 struct vhdr_crc64 *vh = hdr_priv(hdr); 631 632 vh->crc64 = crc64(p, len); 633} 634 635static void fill_md5(struct verify_header *hdr, void *p, unsigned int len) 636{ 637 struct vhdr_md5 *vh = hdr_priv(hdr); 638 struct md5_ctx md5_ctx = { 639 .hash = (uint32_t *) vh->md5_digest, 640 }; 641 642 md5_init(&md5_ctx); 643 md5_update(&md5_ctx, p, len); 644} 645 646/* 647 * fill body of io_u->buf with random data and add a header with the 648 * crc32 or md5 sum of that data. 649 */ 650void populate_verify_io_u(struct thread_data *td, struct io_u *io_u) 651{ 652 struct verify_header *hdr; 653 void *p = io_u->buf, *data; 654 unsigned int hdr_inc, data_len, header_num = 0; 655 656 if (td->o.verify == VERIFY_NULL) 657 return; 658 659 fill_pattern(td, p, io_u->buflen); 660 661 hdr_inc = io_u->buflen; 662 if (td->o.verify_interval) 663 hdr_inc = td->o.verify_interval; 664 665 for (; p < io_u->buf + io_u->buflen; p += hdr_inc) { 666 hdr = p; 667 668 hdr->fio_magic = FIO_HDR_MAGIC; 669 hdr->verify_type = td->o.verify; 670 hdr->len = hdr_inc; 671 data_len = hdr_inc - hdr_size(hdr); 672 673 data = p + hdr_size(hdr); 674 switch (td->o.verify) { 675 case VERIFY_MD5: 676 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n", 677 io_u, hdr->len); 678 fill_md5(hdr, data, data_len); 679 break; 680 case VERIFY_CRC64: 681 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n", 682 io_u, hdr->len); 683 fill_crc64(hdr, data, data_len); 684 break; 685 case VERIFY_CRC32C: 686 case VERIFY_CRC32C_INTEL: 687 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n", 688 io_u, hdr->len); 689 fill_crc32c(hdr, data, data_len); 690 break; 691 case VERIFY_CRC32: 692 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n", 693 io_u, hdr->len); 694 fill_crc32(hdr, data, data_len); 695 break; 696 case VERIFY_CRC16: 697 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n", 698 io_u, hdr->len); 699 fill_crc16(hdr, data, data_len); 700 break; 701 case VERIFY_CRC7: 702 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n", 703 io_u, hdr->len); 704 fill_crc7(hdr, data, data_len); 705 break; 706 case VERIFY_SHA256: 707 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n", 708 io_u, hdr->len); 709 fill_sha256(hdr, data, data_len); 710 break; 711 case VERIFY_SHA512: 712 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n", 713 io_u, hdr->len); 714 fill_sha512(hdr, data, data_len); 715 break; 716 case VERIFY_META: 717 dprint(FD_VERIFY, "fill meta io_u %p, len %u\n", 718 io_u, hdr->len); 719 fill_meta(hdr, td, io_u, header_num); 720 break; 721 case VERIFY_SHA1: 722 dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n", 723 io_u, hdr->len); 724 fill_sha1(hdr, data, data_len); 725 break; 726 default: 727 log_err("fio: bad verify type: %d\n", td->o.verify); 728 assert(0); 729 } 730 if (td->o.verify_offset) 731 memswp(p, p + td->o.verify_offset, hdr_size(hdr)); 732 header_num++; 733 } 734} 735 736int get_next_verify(struct thread_data *td, struct io_u *io_u) 737{ 738 struct io_piece *ipo = NULL; 739 740 /* 741 * this io_u is from a requeue, we already filled the offsets 742 */ 743 if (io_u->file) 744 return 0; 745 746 if (!RB_EMPTY_ROOT(&td->io_hist_tree)) { 747 struct rb_node *n = rb_first(&td->io_hist_tree); 748 749 ipo = rb_entry(n, struct io_piece, rb_node); 750 rb_erase(n, &td->io_hist_tree); 751 td->io_hist_len--; 752 } else if (!flist_empty(&td->io_hist_list)) { 753 ipo = flist_entry(td->io_hist_list.next, struct io_piece, list); 754 td->io_hist_len--; 755 flist_del(&ipo->list); 756 } 757 758 if (ipo) { 759 io_u->offset = ipo->offset; 760 io_u->buflen = ipo->len; 761 io_u->file = ipo->file; 762 763 if (!fio_file_open(io_u->file)) { 764 int r = td_io_open_file(td, io_u->file); 765 766 if (r) { 767 dprint(FD_VERIFY, "failed file %s open\n", 768 io_u->file->file_name); 769 return 1; 770 } 771 } 772 773 get_file(ipo->file); 774 assert(fio_file_open(io_u->file)); 775 io_u->ddir = DDIR_READ; 776 io_u->xfer_buf = io_u->buf; 777 io_u->xfer_buflen = io_u->buflen; 778 free(ipo); 779 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u); 780 return 0; 781 } 782 783 dprint(FD_VERIFY, "get_next_verify: empty\n"); 784 return 1; 785} 786 787static void *verify_async_thread(void *data) 788{ 789 struct thread_data *td = data; 790 struct io_u *io_u; 791 int ret = 0; 792 793 if (td->o.verify_cpumask_set && 794 fio_setaffinity(td->pid, td->o.verify_cpumask)) { 795 log_err("fio: failed setting verify thread affinity\n"); 796 goto done; 797 } 798 799 do { 800 FLIST_HEAD(list); 801 802 read_barrier(); 803 if (td->verify_thread_exit) 804 break; 805 806 pthread_mutex_lock(&td->io_u_lock); 807 808 while (flist_empty(&td->verify_list) && 809 !td->verify_thread_exit) { 810 ret = pthread_cond_wait(&td->verify_cond, 811 &td->io_u_lock); 812 if (ret) { 813 pthread_mutex_unlock(&td->io_u_lock); 814 break; 815 } 816 } 817 818 flist_splice_init(&td->verify_list, &list); 819 pthread_mutex_unlock(&td->io_u_lock); 820 821 if (flist_empty(&list)) 822 continue; 823 824 while (!flist_empty(&list)) { 825 io_u = flist_entry(list.next, struct io_u, list); 826 flist_del_init(&io_u->list); 827 828 ret = verify_io_u(td, io_u); 829 put_io_u(td, io_u); 830 if (!ret) 831 continue; 832 if (td->o.continue_on_error && 833 td_non_fatal_error(ret)) { 834 update_error_count(td, ret); 835 td_clear_error(td); 836 ret = 0; 837 } 838 } 839 } while (!ret); 840 841 if (ret) { 842 td_verror(td, ret, "async_verify"); 843 td->terminate = 1; 844 } 845 846done: 847 pthread_mutex_lock(&td->io_u_lock); 848 td->nr_verify_threads--; 849 pthread_mutex_unlock(&td->io_u_lock); 850 851 pthread_cond_signal(&td->free_cond); 852 return NULL; 853} 854 855int verify_async_init(struct thread_data *td) 856{ 857 int i, ret; 858 859 td->verify_thread_exit = 0; 860 861 td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async); 862 for (i = 0; i < td->o.verify_async; i++) { 863 ret = pthread_create(&td->verify_threads[i], NULL, 864 verify_async_thread, td); 865 if (ret) { 866 log_err("fio: async verify creation failed: %s\n", 867 strerror(ret)); 868 break; 869 } 870 ret = pthread_detach(td->verify_threads[i]); 871 if (ret) { 872 log_err("fio: async verify thread detach failed: %s\n", 873 strerror(ret)); 874 break; 875 } 876 td->nr_verify_threads++; 877 } 878 879 if (i != td->o.verify_async) { 880 log_err("fio: only %d verify threads started, exiting\n", i); 881 td->verify_thread_exit = 1; 882 write_barrier(); 883 pthread_cond_broadcast(&td->verify_cond); 884 return 1; 885 } 886 887 return 0; 888} 889 890void verify_async_exit(struct thread_data *td) 891{ 892 td->verify_thread_exit = 1; 893 write_barrier(); 894 pthread_cond_broadcast(&td->verify_cond); 895 896 pthread_mutex_lock(&td->io_u_lock); 897 898 while (td->nr_verify_threads) 899 pthread_cond_wait(&td->free_cond, &td->io_u_lock); 900 901 pthread_mutex_unlock(&td->io_u_lock); 902 free(td->verify_threads); 903 td->verify_threads = NULL; 904} 905