verify.c revision b36e298b5bdfc31ffc9a4c14e94a9c5c748888b0
1/* 2 * IO verification helpers 3 */ 4#include <unistd.h> 5#include <fcntl.h> 6#include <string.h> 7#include <assert.h> 8#include <pthread.h> 9 10#include "fio.h" 11#include "verify.h" 12#include "smalloc.h" 13 14#include "crc/md5.h" 15#include "crc/crc64.h" 16#include "crc/crc32.h" 17#include "crc/crc32c.h" 18#include "crc/crc16.h" 19#include "crc/crc7.h" 20#include "crc/sha256.h" 21#include "crc/sha512.h" 22 23static void fill_random_bytes(struct thread_data *td, void *p, unsigned int len) 24{ 25 unsigned int todo; 26 int r; 27 28 while (len) { 29 r = os_random_long(&td->verify_state); 30 31 /* 32 * lrand48_r seems to be broken and only fill the bottom 33 * 32-bits, even on 64-bit archs with 64-bit longs 34 */ 35 todo = sizeof(r); 36 if (todo > len) 37 todo = len; 38 39 memcpy(p, &r, todo); 40 41 len -= todo; 42 p += todo; 43 } 44} 45 46static void fill_pattern(struct thread_data *td, void *p, unsigned int len) 47{ 48 switch (td->o.verify_pattern_bytes) { 49 case 0: 50 dprint(FD_VERIFY, "fill random bytes len=%u\n", len); 51 fill_random_bytes(td, p, len); 52 break; 53 case 1: 54 dprint(FD_VERIFY, "fill verify pattern b=0 len=%u\n", len); 55 memset(p, td->o.verify_pattern, len); 56 break; 57 case 2: 58 case 3: 59 case 4: { 60 unsigned int pattern = td->o.verify_pattern; 61 unsigned int i = 0; 62 unsigned char c1, c2, c3, c4; 63 unsigned char *b = p; 64 65 dprint(FD_VERIFY, "fill verify pattern b=%d len=%u\n", 66 td->o.verify_pattern_bytes, len); 67 68 c1 = pattern & 0xff; 69 pattern >>= 8; 70 c2 = pattern & 0xff; 71 pattern >>= 8; 72 c3 = pattern & 0xff; 73 pattern >>= 8; 74 c4 = pattern & 0xff; 75 76 while (i < len) { 77 b[i++] = c1; 78 if (i == len) 79 break; 80 b[i++] = c2; 81 if (td->o.verify_pattern_bytes == 2 || i == len) 82 continue; 83 b[i++] = c3; 84 if (td->o.verify_pattern_bytes == 3 || i == len) 85 continue; 86 b[i++] = c4; 87 } 88 break; 89 } 90 } 91} 92 93static void memswp(void *buf1, void *buf2, unsigned int len) 94{ 95 char swap[200]; 96 97 assert(len <= sizeof(swap)); 98 99 memcpy(&swap, buf1, len); 100 memcpy(buf1, buf2, len); 101 memcpy(buf2, &swap, len); 102} 103 104static void hexdump(void *buffer, int len) 105{ 106 unsigned char *p = buffer; 107 int i; 108 109 for (i = 0; i < len; i++) 110 log_info("%02x", p[i]); 111 log_info("\n"); 112} 113 114/* 115 * Prepare for seperation of verify_header and checksum header 116 */ 117static inline unsigned int __hdr_size(int verify_type) 118{ 119 unsigned int len = len; 120 121 switch (verify_type) { 122 case VERIFY_NONE: 123 case VERIFY_NULL: 124 len = 0; 125 break; 126 case VERIFY_MD5: 127 len = sizeof(struct vhdr_md5); 128 break; 129 case VERIFY_CRC64: 130 len = sizeof(struct vhdr_crc64); 131 break; 132 case VERIFY_CRC32C: 133 case VERIFY_CRC32: 134 case VERIFY_CRC32C_INTEL: 135 len = sizeof(struct vhdr_crc32); 136 break; 137 case VERIFY_CRC16: 138 len = sizeof(struct vhdr_crc16); 139 break; 140 case VERIFY_CRC7: 141 len = sizeof(struct vhdr_crc7); 142 break; 143 case VERIFY_SHA256: 144 len = sizeof(struct vhdr_sha256); 145 break; 146 case VERIFY_SHA512: 147 len = sizeof(struct vhdr_sha512); 148 break; 149 case VERIFY_META: 150 len = sizeof(struct vhdr_meta); 151 break; 152 default: 153 log_err("fio: unknown verify header!\n"); 154 assert(0); 155 } 156 157 return len + sizeof(struct verify_header); 158} 159 160static inline unsigned int hdr_size(struct verify_header *hdr) 161{ 162 return __hdr_size(hdr->verify_type); 163} 164 165static void *hdr_priv(struct verify_header *hdr) 166{ 167 void *priv = hdr; 168 169 return priv + sizeof(struct verify_header); 170} 171 172/* 173 * Return data area 'header_num' 174 */ 175static inline void *io_u_verify_off(struct verify_header *hdr, 176 struct io_u *io_u, unsigned char header_num) 177{ 178 return io_u->buf + header_num * hdr->len + hdr_size(hdr); 179} 180 181static int verify_io_u_meta(struct verify_header *hdr, struct thread_data *td, 182 struct io_u *io_u, unsigned int header_num) 183{ 184 struct vhdr_meta *vh = hdr_priv(hdr); 185 186 dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len); 187 188 if (vh->offset != io_u->offset + header_num * td->o.verify_interval) { 189 log_err("meta: verify failed at %llu/%u\n", 190 io_u->offset + header_num * hdr->len, hdr->len); 191 return EILSEQ; 192 } 193 194 return 0; 195} 196 197static int verify_io_u_sha512(struct verify_header *hdr, struct io_u *io_u, 198 unsigned int header_num) 199{ 200 void *p = io_u_verify_off(hdr, io_u, header_num); 201 struct vhdr_sha512 *vh = hdr_priv(hdr); 202 uint8_t sha512[128]; 203 struct sha512_ctx sha512_ctx = { 204 .buf = sha512, 205 }; 206 207 dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", io_u, hdr->len); 208 209 sha512_init(&sha512_ctx); 210 sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr)); 211 212 if (memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512))) { 213 log_err("sha512: verify failed at %llu/%u\n", 214 io_u->offset + header_num * hdr->len, hdr->len); 215 hexdump(vh->sha512, sizeof(vh->sha512)); 216 hexdump(sha512_ctx.buf, sizeof(sha512)); 217 return EILSEQ; 218 } 219 220 return 0; 221} 222 223static int verify_io_u_sha256(struct verify_header *hdr, struct io_u *io_u, 224 unsigned int header_num) 225{ 226 void *p = io_u_verify_off(hdr, io_u, header_num); 227 struct vhdr_sha256 *vh = hdr_priv(hdr); 228 uint8_t sha256[128]; 229 struct sha256_ctx sha256_ctx = { 230 .buf = sha256, 231 }; 232 233 dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", io_u, hdr->len); 234 235 sha256_init(&sha256_ctx); 236 sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr)); 237 238 if (memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256))) { 239 log_err("sha256: verify failed at %llu/%u\n", 240 io_u->offset + header_num * hdr->len, hdr->len); 241 hexdump(vh->sha256, sizeof(vh->sha256)); 242 hexdump(sha256_ctx.buf, sizeof(sha256)); 243 return EILSEQ; 244 } 245 246 return 0; 247} 248 249static int verify_io_u_crc7(struct verify_header *hdr, struct io_u *io_u, 250 unsigned char header_num) 251{ 252 void *p = io_u_verify_off(hdr, io_u, header_num); 253 struct vhdr_crc7 *vh = hdr_priv(hdr); 254 unsigned char c; 255 256 dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", io_u, hdr->len); 257 258 c = crc7(p, hdr->len - hdr_size(hdr)); 259 260 if (c != vh->crc7) { 261 log_err("crc7: verify failed at %llu/%u\n", 262 io_u->offset + header_num * hdr->len, hdr->len); 263 log_err("crc7: wanted %x, got %x\n", vh->crc7, c); 264 return EILSEQ; 265 } 266 267 return 0; 268} 269 270static int verify_io_u_crc16(struct verify_header *hdr, struct io_u *io_u, 271 unsigned int header_num) 272{ 273 void *p = io_u_verify_off(hdr, io_u, header_num); 274 struct vhdr_crc16 *vh = hdr_priv(hdr); 275 unsigned short c; 276 277 dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", io_u, hdr->len); 278 279 c = crc16(p, hdr->len - hdr_size(hdr)); 280 281 if (c != vh->crc16) { 282 log_err("crc16: verify failed at %llu/%u\n", 283 io_u->offset + header_num * hdr->len, hdr->len); 284 log_err("crc16: wanted %x, got %x\n", vh->crc16, c); 285 return EILSEQ; 286 } 287 288 return 0; 289} 290 291static int verify_io_u_crc64(struct verify_header *hdr, struct io_u *io_u, 292 unsigned int header_num) 293{ 294 void *p = io_u_verify_off(hdr, io_u, header_num); 295 struct vhdr_crc64 *vh = hdr_priv(hdr); 296 unsigned long long c; 297 298 dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", io_u, hdr->len); 299 300 c = crc64(p, hdr->len - hdr_size(hdr)); 301 302 if (c != vh->crc64) { 303 log_err("crc64: verify failed at %llu/%u\n", 304 io_u->offset + header_num * hdr->len, 305 hdr->len); 306 log_err("crc64: wanted %llx, got %llx\n", 307 (unsigned long long) vh->crc64, c); 308 return EILSEQ; 309 } 310 311 return 0; 312} 313 314static int verify_io_u_crc32(struct verify_header *hdr, struct io_u *io_u, 315 unsigned int header_num) 316{ 317 void *p = io_u_verify_off(hdr, io_u, header_num); 318 struct vhdr_crc32 *vh = hdr_priv(hdr); 319 uint32_t c; 320 321 dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", io_u, hdr->len); 322 323 c = crc32(p, hdr->len - hdr_size(hdr)); 324 325 if (c != vh->crc32) { 326 log_err("crc32: verify failed at %llu/%u\n", 327 io_u->offset + header_num * hdr->len, hdr->len); 328 log_err("crc32: wanted %x, got %x\n", vh->crc32, c); 329 return EILSEQ; 330 } 331 332 return 0; 333} 334 335static int verify_io_u_crc32c(struct verify_header *hdr, struct io_u *io_u, 336 unsigned int header_num) 337{ 338 void *p = io_u_verify_off(hdr, io_u, header_num); 339 struct vhdr_crc32 *vh = hdr_priv(hdr); 340 uint32_t c; 341 342 dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", io_u, hdr->len); 343 344 if (hdr->verify_type == VERIFY_CRC32C_INTEL) 345 c = crc32c_intel(p, hdr->len - hdr_size(hdr)); 346 else 347 c = crc32c(p, hdr->len - hdr_size(hdr)); 348 349 if (c != vh->crc32) { 350 log_err("crc32c: verify failed at %llu/%u\n", 351 io_u->offset + header_num * hdr->len, hdr->len); 352 log_err("crc32c: wanted %x, got %x\n", vh->crc32, c); 353 return EILSEQ; 354 } 355 356 return 0; 357} 358 359static int verify_io_u_md5(struct verify_header *hdr, struct io_u *io_u, 360 unsigned int header_num) 361{ 362 void *p = io_u_verify_off(hdr, io_u, header_num); 363 struct vhdr_md5 *vh = hdr_priv(hdr); 364 uint32_t hash[MD5_HASH_WORDS]; 365 struct md5_ctx md5_ctx = { 366 .hash = hash, 367 }; 368 369 dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", io_u, hdr->len); 370 371 md5_init(&md5_ctx); 372 md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr)); 373 374 if (memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash))) { 375 log_err("md5: verify failed at %llu/%u\n", 376 io_u->offset + header_num * hdr->len, hdr->len); 377 hexdump(vh->md5_digest, sizeof(vh->md5_digest)); 378 hexdump(md5_ctx.hash, sizeof(hash)); 379 return EILSEQ; 380 } 381 382 return 0; 383} 384 385static unsigned int hweight8(unsigned int w) 386{ 387 unsigned int res = w - ((w >> 1) & 0x55); 388 389 res = (res & 0x33) + ((res >> 2) & 0x33); 390 return (res + (res >> 4)) & 0x0F; 391} 392 393int verify_io_u_pattern(unsigned long pattern, unsigned long pattern_size, 394 char *buf, unsigned int len, unsigned int mod) 395{ 396 unsigned int i; 397 char split_pattern[4]; 398 399 for (i = 0; i < 4; i++) { 400 split_pattern[i] = pattern & 0xff; 401 pattern >>= 8; 402 } 403 404 for (i = 0; i < len; i++) { 405 if (buf[i] != split_pattern[mod]) { 406 unsigned int bits; 407 408 bits = hweight8(buf[i] ^ split_pattern[mod]); 409 log_err("fio: got pattern %x, wanted %x. Bad bits %d\n", 410 buf[i], split_pattern[mod], bits); 411 log_err("fio: bad pattern block offset %u\n", i); 412 return EILSEQ; 413 } 414 mod++; 415 if (mod == pattern_size) 416 mod = 0; 417 } 418 419 return 0; 420} 421 422/* 423 * Push IO verification to a separate thread 424 */ 425int verify_io_u_async(struct thread_data *td, struct io_u *io_u) 426{ 427 if (io_u->file) 428 put_file_log(td, io_u->file); 429 430 io_u->file = NULL; 431 432 pthread_mutex_lock(&td->io_u_lock); 433 flist_del(&io_u->list); 434 flist_add_tail(&io_u->list, &td->verify_list); 435 pthread_mutex_unlock(&td->io_u_lock); 436 437 pthread_cond_signal(&td->verify_cond); 438 io_u->flags |= IO_U_F_FREE_DEF; 439 return 0; 440} 441 442int verify_io_u(struct thread_data *td, struct io_u *io_u) 443{ 444 struct verify_header *hdr; 445 unsigned int hdr_size, hdr_inc, hdr_num = 0; 446 void *p; 447 int ret; 448 449 if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ) 450 return 0; 451 452 hdr_inc = io_u->buflen; 453 if (td->o.verify_interval) 454 hdr_inc = td->o.verify_interval; 455 456 ret = 0; 457 for (p = io_u->buf; p < io_u->buf + io_u->buflen; 458 p += hdr_inc, hdr_num++) { 459 if (ret && td->o.verify_fatal) { 460 td->terminate = 1; 461 break; 462 } 463 hdr_size = __hdr_size(td->o.verify); 464 if (td->o.verify_offset) 465 memswp(p, p + td->o.verify_offset, hdr_size); 466 hdr = p; 467 468 if (hdr->fio_magic != FIO_HDR_MAGIC) { 469 log_err("Bad verify header %x\n", hdr->fio_magic); 470 return EILSEQ; 471 } 472 473 if (td->o.verify_pattern_bytes) { 474 dprint(FD_VERIFY, "pattern verify io_u %p, len %u\n", 475 io_u, hdr->len); 476 ret = verify_io_u_pattern(td->o.verify_pattern, 477 td->o.verify_pattern_bytes, 478 p + hdr_size, 479 hdr_inc - hdr_size, 480 hdr_size % 4); 481 if (ret) 482 log_err("fio: verify failed at %llu/%u\n", 483 io_u->offset + hdr_num * hdr->len, 484 hdr->len); 485 continue; 486 } 487 488 switch (hdr->verify_type) { 489 case VERIFY_MD5: 490 ret = verify_io_u_md5(hdr, io_u, hdr_num); 491 break; 492 case VERIFY_CRC64: 493 ret = verify_io_u_crc64(hdr, io_u, hdr_num); 494 break; 495 case VERIFY_CRC32C: 496 case VERIFY_CRC32C_INTEL: 497 ret = verify_io_u_crc32c(hdr, io_u, hdr_num); 498 break; 499 case VERIFY_CRC32: 500 ret = verify_io_u_crc32(hdr, io_u, hdr_num); 501 break; 502 case VERIFY_CRC16: 503 ret = verify_io_u_crc16(hdr, io_u, hdr_num); 504 break; 505 case VERIFY_CRC7: 506 ret = verify_io_u_crc7(hdr, io_u, hdr_num); 507 break; 508 case VERIFY_SHA256: 509 ret = verify_io_u_sha256(hdr, io_u, hdr_num); 510 break; 511 case VERIFY_SHA512: 512 ret = verify_io_u_sha512(hdr, io_u, hdr_num); 513 break; 514 case VERIFY_META: 515 ret = verify_io_u_meta(hdr, td, io_u, hdr_num); 516 break; 517 default: 518 log_err("Bad verify type %u\n", hdr->verify_type); 519 ret = EINVAL; 520 } 521 } 522 523 return ret; 524} 525 526static void fill_meta(struct verify_header *hdr, struct thread_data *td, 527 struct io_u *io_u, unsigned int header_num) 528{ 529 struct vhdr_meta *vh = hdr_priv(hdr); 530 531 vh->thread = td->thread_number; 532 533 vh->time_sec = io_u->start_time.tv_sec; 534 vh->time_usec = io_u->start_time.tv_usec; 535 536 vh->numberio = td->io_issues[DDIR_WRITE]; 537 538 vh->offset = io_u->offset + header_num * td->o.verify_interval; 539} 540 541static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len) 542{ 543 struct vhdr_sha512 *vh = hdr_priv(hdr); 544 struct sha512_ctx sha512_ctx = { 545 .buf = vh->sha512, 546 }; 547 548 sha512_init(&sha512_ctx); 549 sha512_update(&sha512_ctx, p, len); 550} 551 552static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len) 553{ 554 struct vhdr_sha256 *vh = hdr_priv(hdr); 555 struct sha256_ctx sha256_ctx = { 556 .buf = vh->sha256, 557 }; 558 559 sha256_init(&sha256_ctx); 560 sha256_update(&sha256_ctx, p, len); 561} 562 563static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len) 564{ 565 struct vhdr_crc7 *vh = hdr_priv(hdr); 566 567 vh->crc7 = crc7(p, len); 568} 569 570static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len) 571{ 572 struct vhdr_crc16 *vh = hdr_priv(hdr); 573 574 vh->crc16 = crc16(p, len); 575} 576 577static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len) 578{ 579 struct vhdr_crc32 *vh = hdr_priv(hdr); 580 581 vh->crc32 = crc32(p, len); 582} 583 584static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len) 585{ 586 struct vhdr_crc32 *vh = hdr_priv(hdr); 587 588 if (hdr->verify_type == VERIFY_CRC32C_INTEL) 589 vh->crc32 = crc32c_intel(p, len); 590 else 591 vh->crc32 = crc32c(p, len); 592} 593 594static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len) 595{ 596 struct vhdr_crc64 *vh = hdr_priv(hdr); 597 598 vh->crc64 = crc64(p, len); 599} 600 601static void fill_md5(struct verify_header *hdr, void *p, unsigned int len) 602{ 603 struct vhdr_md5 *vh = hdr_priv(hdr); 604 struct md5_ctx md5_ctx = { 605 .hash = (uint32_t *) vh->md5_digest, 606 }; 607 608 md5_init(&md5_ctx); 609 md5_update(&md5_ctx, p, len); 610} 611 612/* 613 * fill body of io_u->buf with random data and add a header with the 614 * crc32 or md5 sum of that data. 615 */ 616void populate_verify_io_u(struct thread_data *td, struct io_u *io_u) 617{ 618 struct verify_header *hdr; 619 void *p = io_u->buf, *data; 620 unsigned int hdr_inc, data_len, header_num = 0; 621 622 if (td->o.verify == VERIFY_NULL) 623 return; 624 625 fill_pattern(td, p, io_u->buflen); 626 627 hdr_inc = io_u->buflen; 628 if (td->o.verify_interval) 629 hdr_inc = td->o.verify_interval; 630 631 for (; p < io_u->buf + io_u->buflen; p += hdr_inc) { 632 hdr = p; 633 634 hdr->fio_magic = FIO_HDR_MAGIC; 635 hdr->verify_type = td->o.verify; 636 hdr->len = hdr_inc; 637 data_len = hdr_inc - hdr_size(hdr); 638 639 data = p + hdr_size(hdr); 640 switch (td->o.verify) { 641 case VERIFY_MD5: 642 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n", 643 io_u, hdr->len); 644 fill_md5(hdr, data, data_len); 645 break; 646 case VERIFY_CRC64: 647 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n", 648 io_u, hdr->len); 649 fill_crc64(hdr, data, data_len); 650 break; 651 case VERIFY_CRC32C: 652 case VERIFY_CRC32C_INTEL: 653 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n", 654 io_u, hdr->len); 655 fill_crc32c(hdr, data, data_len); 656 break; 657 case VERIFY_CRC32: 658 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n", 659 io_u, hdr->len); 660 fill_crc32(hdr, data, data_len); 661 break; 662 case VERIFY_CRC16: 663 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n", 664 io_u, hdr->len); 665 fill_crc16(hdr, data, data_len); 666 break; 667 case VERIFY_CRC7: 668 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n", 669 io_u, hdr->len); 670 fill_crc7(hdr, data, data_len); 671 break; 672 case VERIFY_SHA256: 673 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n", 674 io_u, hdr->len); 675 fill_sha256(hdr, data, data_len); 676 break; 677 case VERIFY_SHA512: 678 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n", 679 io_u, hdr->len); 680 fill_sha512(hdr, data, data_len); 681 break; 682 case VERIFY_META: 683 dprint(FD_VERIFY, "fill meta io_u %p, len %u\n", 684 io_u, hdr->len); 685 fill_meta(hdr, td, io_u, header_num); 686 break; 687 default: 688 log_err("fio: bad verify type: %d\n", td->o.verify); 689 assert(0); 690 } 691 if (td->o.verify_offset) 692 memswp(p, p + td->o.verify_offset, hdr_size(hdr)); 693 header_num++; 694 } 695} 696 697int get_next_verify(struct thread_data *td, struct io_u *io_u) 698{ 699 struct io_piece *ipo = NULL; 700 701 /* 702 * this io_u is from a requeue, we already filled the offsets 703 */ 704 if (io_u->file) 705 return 0; 706 707 if (!RB_EMPTY_ROOT(&td->io_hist_tree)) { 708 struct rb_node *n = rb_first(&td->io_hist_tree); 709 710 ipo = rb_entry(n, struct io_piece, rb_node); 711 rb_erase(n, &td->io_hist_tree); 712 } else if (!flist_empty(&td->io_hist_list)) { 713 ipo = flist_entry(td->io_hist_list.next, struct io_piece, list); 714 flist_del(&ipo->list); 715 } 716 717 if (ipo) { 718 io_u->offset = ipo->offset; 719 io_u->buflen = ipo->len; 720 io_u->file = ipo->file; 721 722 if (!fio_file_open(io_u->file)) { 723 int r = td_io_open_file(td, io_u->file); 724 725 if (r) { 726 dprint(FD_VERIFY, "failed file %s open\n", 727 io_u->file->file_name); 728 return 1; 729 } 730 } 731 732 get_file(ipo->file); 733 assert(fio_file_open(io_u->file)); 734 io_u->ddir = DDIR_READ; 735 io_u->xfer_buf = io_u->buf; 736 io_u->xfer_buflen = io_u->buflen; 737 free(ipo); 738 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u); 739 return 0; 740 } 741 742 dprint(FD_VERIFY, "get_next_verify: empty\n"); 743 return 1; 744} 745 746static void *verify_async_thread(void *data) 747{ 748 struct thread_data *td = data; 749 struct io_u *io_u; 750 int ret = 0; 751 752 if (td->o.verify_cpumask_set && 753 fio_setaffinity(td->pid, td->o.verify_cpumask)) { 754 log_err("fio: failed setting verify thread affinity\n"); 755 goto done; 756 } 757 758 do { 759 FLIST_HEAD(list); 760 761 read_barrier(); 762 if (td->verify_thread_exit) 763 break; 764 765 pthread_mutex_lock(&td->io_u_lock); 766 767 while (flist_empty(&td->verify_list) && 768 !td->verify_thread_exit) { 769 ret = pthread_cond_wait(&td->verify_cond, 770 &td->io_u_lock); 771 if (ret) { 772 pthread_mutex_unlock(&td->io_u_lock); 773 break; 774 } 775 } 776 777 flist_splice_init(&td->verify_list, &list); 778 pthread_mutex_unlock(&td->io_u_lock); 779 780 if (flist_empty(&list)) 781 continue; 782 783 while (!flist_empty(&list)) { 784 io_u = flist_entry(list.next, struct io_u, list); 785 flist_del_init(&io_u->list); 786 787 ret = verify_io_u(td, io_u); 788 put_io_u(td, io_u); 789 if (!ret) 790 continue; 791 if (td->o.continue_on_error && 792 td_non_fatal_error(ret)) { 793 update_error_count(td, ret); 794 td_clear_error(td); 795 ret = 0; 796 } 797 } 798 } while (!ret); 799 800 if (ret) { 801 td_verror(td, ret, "async_verify"); 802 td->terminate = 1; 803 } 804 805done: 806 pthread_mutex_lock(&td->io_u_lock); 807 td->nr_verify_threads--; 808 pthread_mutex_unlock(&td->io_u_lock); 809 810 pthread_cond_signal(&td->free_cond); 811 return NULL; 812} 813 814int verify_async_init(struct thread_data *td) 815{ 816 int i, ret; 817 818 td->verify_thread_exit = 0; 819 820 td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async); 821 for (i = 0; i < td->o.verify_async; i++) { 822 ret = pthread_create(&td->verify_threads[i], NULL, 823 verify_async_thread, td); 824 if (ret) { 825 log_err("fio: async verify creation failed: %s\n", 826 strerror(ret)); 827 break; 828 } 829 ret = pthread_detach(td->verify_threads[i]); 830 if (ret) { 831 log_err("fio: async verify thread detach failed: %s\n", 832 strerror(ret)); 833 break; 834 } 835 td->nr_verify_threads++; 836 } 837 838 if (i != td->o.verify_async) { 839 td->verify_thread_exit = 1; 840 write_barrier(); 841 pthread_cond_broadcast(&td->verify_cond); 842 return 1; 843 } 844 845 return 0; 846} 847 848void verify_async_exit(struct thread_data *td) 849{ 850 td->verify_thread_exit = 1; 851 write_barrier(); 852 pthread_cond_broadcast(&td->verify_cond); 853 854 pthread_mutex_lock(&td->io_u_lock); 855 856 while (td->nr_verify_threads) 857 pthread_cond_wait(&td->free_cond, &td->io_u_lock); 858 859 pthread_mutex_unlock(&td->io_u_lock); 860 free(td->verify_threads); 861 td->verify_threads = NULL; 862} 863