verify.c revision 3845591fadea480177223e28c9d1c03642d34f0e
1/* 2 * IO verification helpers 3 */ 4#include <unistd.h> 5#include <fcntl.h> 6#include <string.h> 7#include <assert.h> 8 9#include "fio.h" 10 11#include "crc/md5.h" 12#include "crc/crc64.h" 13#include "crc/crc32.h" 14#include "crc/crc32c.h" 15#include "crc/crc16.h" 16#include "crc/crc7.h" 17#include "crc/sha256.h" 18#include "crc/sha512.h" 19 20static void fill_random_bytes(struct thread_data *td, void *p, unsigned int len) 21{ 22 unsigned int todo; 23 int r; 24 25 while (len) { 26 r = os_random_long(&td->verify_state); 27 28 /* 29 * lrand48_r seems to be broken and only fill the bottom 30 * 32-bits, even on 64-bit archs with 64-bit longs 31 */ 32 todo = sizeof(r); 33 if (todo > len) 34 todo = len; 35 36 memcpy(p, &r, todo); 37 38 len -= todo; 39 p += todo; 40 } 41} 42 43static void fill_pattern(struct thread_data *td, void *p, unsigned int len) 44{ 45 switch (td->o.verify_pattern_bytes) { 46 case 0: 47 dprint(FD_VERIFY, "fill random bytes len=%u\n", len); 48 fill_random_bytes(td, p, len); 49 break; 50 case 1: 51 dprint(FD_VERIFY, "fill verify pattern b=0 len=%u\n", len); 52 memset(p, td->o.verify_pattern, len); 53 break; 54 case 2: 55 case 3: 56 case 4: { 57 unsigned int pattern = td->o.verify_pattern; 58 unsigned int i = 0; 59 unsigned char c1, c2, c3, c4; 60 unsigned char *b = p; 61 62 dprint(FD_VERIFY, "fill verify pattern b=%d len=%u\n", 63 td->o.verify_pattern_bytes, len); 64 65 c1 = pattern & 0xff; 66 pattern >>= 8; 67 c2 = pattern & 0xff; 68 pattern >>= 8; 69 c3 = pattern & 0xff; 70 pattern >>= 8; 71 c4 = pattern & 0xff; 72 73 while (i < len) { 74 b[i++] = c1; 75 if (i == len) 76 break; 77 b[i++] = c2; 78 if (td->o.verify_pattern_bytes == 2 || i == len) 79 continue; 80 b[i++] = c3; 81 if (td->o.verify_pattern_bytes == 3 || i == len) 82 continue; 83 b[i++] = c4; 84 } 85 break; 86 } 87 } 88} 89 90static void memswp(void *buf1, void *buf2, unsigned int len) 91{ 92 char swap[200]; 93 94 assert(len <= sizeof(swap)); 95 96 memcpy(&swap, buf1, len); 97 memcpy(buf1, buf2, len); 98 memcpy(buf2, &swap, len); 99} 100 101static void hexdump(void *buffer, int len) 102{ 103 unsigned char *p = buffer; 104 int i; 105 106 for (i = 0; i < len; i++) 107 log_info("%02x", p[i]); 108 log_info("\n"); 109} 110 111/* 112 * Prepare for seperation of verify_header and checksum header 113 */ 114static inline unsigned int __hdr_size(int verify_type) 115{ 116 unsigned int len = len; 117 118 switch (verify_type) { 119 case VERIFY_NONE: 120 case VERIFY_NULL: 121 len = 0; 122 break; 123 case VERIFY_MD5: 124 len = sizeof(struct vhdr_md5); 125 break; 126 case VERIFY_CRC64: 127 len = sizeof(struct vhdr_crc64); 128 break; 129 case VERIFY_CRC32C: 130 case VERIFY_CRC32: 131 case VERIFY_CRC32C_INTEL: 132 len = sizeof(struct vhdr_crc32); 133 break; 134 case VERIFY_CRC16: 135 len = sizeof(struct vhdr_crc16); 136 break; 137 case VERIFY_CRC7: 138 len = sizeof(struct vhdr_crc7); 139 break; 140 case VERIFY_SHA256: 141 len = sizeof(struct vhdr_sha256); 142 break; 143 case VERIFY_SHA512: 144 len = sizeof(struct vhdr_sha512); 145 break; 146 case VERIFY_META: 147 len = sizeof(struct vhdr_meta); 148 break; 149 default: 150 log_err("fio: unknown verify header!\n"); 151 assert(0); 152 } 153 154 return len + sizeof(struct verify_header); 155} 156 157static inline unsigned int hdr_size(struct verify_header *hdr) 158{ 159 return __hdr_size(hdr->verify_type); 160} 161 162static void *hdr_priv(struct verify_header *hdr) 163{ 164 void *priv = hdr; 165 166 return priv + sizeof(struct verify_header); 167} 168 169/* 170 * Return data area 'header_num' 171 */ 172static inline void *io_u_verify_off(struct verify_header *hdr, 173 struct io_u *io_u, unsigned char header_num) 174{ 175 return io_u->buf + header_num * hdr->len + hdr_size(hdr); 176} 177 178static int verify_io_u_meta(struct verify_header *hdr, struct thread_data *td, 179 struct io_u *io_u, unsigned int header_num) 180{ 181 struct vhdr_meta *vh = hdr_priv(hdr); 182 183 dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len); 184 185 if (vh->offset != io_u->offset + header_num * td->o.verify_interval) { 186 log_err("meta: verify failed at %llu/%u\n", 187 io_u->offset + header_num * hdr->len, hdr->len); 188 return EIO; 189 } 190 191 return 0; 192} 193 194static int verify_io_u_sha512(struct verify_header *hdr, struct io_u *io_u, 195 unsigned int header_num) 196{ 197 void *p = io_u_verify_off(hdr, io_u, header_num); 198 struct vhdr_sha512 *vh = hdr_priv(hdr); 199 uint8_t sha512[128]; 200 struct sha512_ctx sha512_ctx = { 201 .buf = sha512, 202 }; 203 204 dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", io_u, hdr->len); 205 206 sha512_init(&sha512_ctx); 207 sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr)); 208 209 if (memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512))) { 210 log_err("sha512: verify failed at %llu/%u\n", 211 io_u->offset + header_num * hdr->len, hdr->len); 212 hexdump(vh->sha512, sizeof(vh->sha512)); 213 hexdump(sha512_ctx.buf, sizeof(sha512)); 214 return EIO; 215 } 216 217 return 0; 218} 219 220static int verify_io_u_sha256(struct verify_header *hdr, struct io_u *io_u, 221 unsigned int header_num) 222{ 223 void *p = io_u_verify_off(hdr, io_u, header_num); 224 struct vhdr_sha256 *vh = hdr_priv(hdr); 225 uint8_t sha256[128]; 226 struct sha256_ctx sha256_ctx = { 227 .buf = sha256, 228 }; 229 230 dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", io_u, hdr->len); 231 232 sha256_init(&sha256_ctx); 233 sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr)); 234 235 if (memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256))) { 236 log_err("sha256: verify failed at %llu/%u\n", 237 io_u->offset + header_num * hdr->len, hdr->len); 238 hexdump(vh->sha256, sizeof(vh->sha256)); 239 hexdump(sha256_ctx.buf, sizeof(sha256)); 240 return EIO; 241 } 242 243 return 0; 244} 245 246static int verify_io_u_crc7(struct verify_header *hdr, struct io_u *io_u, 247 unsigned char header_num) 248{ 249 void *p = io_u_verify_off(hdr, io_u, header_num); 250 struct vhdr_crc7 *vh = hdr_priv(hdr); 251 unsigned char c; 252 253 dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", io_u, hdr->len); 254 255 c = crc7(p, hdr->len - hdr_size(hdr)); 256 257 if (c != vh->crc7) { 258 log_err("crc7: verify failed at %llu/%u\n", 259 io_u->offset + header_num * hdr->len, hdr->len); 260 log_err("crc7: wanted %x, got %x\n", vh->crc7, c); 261 return EIO; 262 } 263 264 return 0; 265} 266 267static int verify_io_u_crc16(struct verify_header *hdr, struct io_u *io_u, 268 unsigned int header_num) 269{ 270 void *p = io_u_verify_off(hdr, io_u, header_num); 271 struct vhdr_crc16 *vh = hdr_priv(hdr); 272 unsigned short c; 273 274 dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", io_u, hdr->len); 275 276 c = crc16(p, hdr->len - hdr_size(hdr)); 277 278 if (c != vh->crc16) { 279 log_err("crc16: verify failed at %llu/%u\n", 280 io_u->offset + header_num * hdr->len, hdr->len); 281 log_err("crc16: wanted %x, got %x\n", vh->crc16, c); 282 return EIO; 283 } 284 285 return 0; 286} 287 288static int verify_io_u_crc64(struct verify_header *hdr, struct io_u *io_u, 289 unsigned int header_num) 290{ 291 void *p = io_u_verify_off(hdr, io_u, header_num); 292 struct vhdr_crc64 *vh = hdr_priv(hdr); 293 unsigned long long c; 294 295 dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", io_u, hdr->len); 296 297 c = crc64(p, hdr->len - hdr_size(hdr)); 298 299 if (c != vh->crc64) { 300 log_err("crc64: verify failed at %llu/%u\n", 301 io_u->offset + header_num * hdr->len, 302 hdr->len); 303 log_err("crc64: wanted %llx, got %llx\n", 304 (unsigned long long) vh->crc64, c); 305 return EIO; 306 } 307 308 return 0; 309} 310 311static int verify_io_u_crc32(struct verify_header *hdr, struct io_u *io_u, 312 unsigned int header_num) 313{ 314 void *p = io_u_verify_off(hdr, io_u, header_num); 315 struct vhdr_crc32 *vh = hdr_priv(hdr); 316 uint32_t c; 317 318 dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", io_u, hdr->len); 319 320 c = crc32(p, hdr->len - hdr_size(hdr)); 321 322 if (c != vh->crc32) { 323 log_err("crc32: verify failed at %llu/%u\n", 324 io_u->offset + header_num * hdr->len, hdr->len); 325 log_err("crc32: wanted %x, got %x\n", vh->crc32, c); 326 return EIO; 327 } 328 329 return 0; 330} 331 332static int verify_io_u_crc32c(struct verify_header *hdr, struct io_u *io_u, 333 unsigned int header_num) 334{ 335 void *p = io_u_verify_off(hdr, io_u, header_num); 336 struct vhdr_crc32 *vh = hdr_priv(hdr); 337 uint32_t c; 338 339 dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", io_u, hdr->len); 340 341 if (hdr->verify_type == VERIFY_CRC32C_INTEL) 342 c = crc32c_intel(p, hdr->len - hdr_size(hdr)); 343 else 344 c = crc32c(p, hdr->len - hdr_size(hdr)); 345 346 if (c != vh->crc32) { 347 log_err("crc32c: verify failed at %llu/%u\n", 348 io_u->offset + header_num * hdr->len, hdr->len); 349 log_err("crc32c: wanted %x, got %x\n", vh->crc32, c); 350 return EIO; 351 } 352 353 return 0; 354} 355 356static int verify_io_u_md5(struct verify_header *hdr, struct io_u *io_u, 357 unsigned int header_num) 358{ 359 void *p = io_u_verify_off(hdr, io_u, header_num); 360 struct vhdr_md5 *vh = hdr_priv(hdr); 361 uint32_t hash[MD5_HASH_WORDS]; 362 struct md5_ctx md5_ctx = { 363 .hash = hash, 364 }; 365 366 dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", io_u, hdr->len); 367 368 md5_init(&md5_ctx); 369 md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr)); 370 371 if (memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash))) { 372 log_err("md5: verify failed at %llu/%u\n", 373 io_u->offset + header_num * hdr->len, hdr->len); 374 hexdump(vh->md5_digest, sizeof(vh->md5_digest)); 375 hexdump(md5_ctx.hash, sizeof(hash)); 376 return EIO; 377 } 378 379 return 0; 380} 381 382static unsigned int hweight8(unsigned int w) 383{ 384 unsigned int res = w - ((w >> 1) & 0x55); 385 386 res = (res & 0x33) + ((res >> 2) & 0x33); 387 return (res + (res >> 4)) & 0x0F; 388} 389 390int verify_io_u_pattern(unsigned long pattern, unsigned long pattern_size, 391 char *buf, unsigned int len, unsigned int mod) 392{ 393 unsigned int i; 394 char split_pattern[4]; 395 396 for (i = 0; i < 4; i++) { 397 split_pattern[i] = pattern & 0xff; 398 pattern >>= 8; 399 } 400 401 for (i = 0; i < len; i++) { 402 if (buf[i] != split_pattern[mod]) { 403 unsigned int bits; 404 405 bits = hweight8(buf[i] ^ split_pattern[mod]); 406 log_err("fio: got pattern %x, wanted %x. Bad bits %d\n", 407 buf[i], split_pattern[mod], bits); 408 log_err("fio: bad pattern block offset %u\n", i); 409 return EIO; 410 } 411 mod++; 412 if (mod == pattern_size) 413 mod = 0; 414 } 415 416 return 0; 417} 418 419int verify_io_u(struct thread_data *td, struct io_u *io_u) 420{ 421 struct verify_header *hdr; 422 unsigned int hdr_size, hdr_inc, hdr_num = 0; 423 void *p; 424 int ret; 425 426 if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ) 427 return 0; 428 429 hdr_inc = io_u->buflen; 430 if (td->o.verify_interval) 431 hdr_inc = td->o.verify_interval; 432 433 ret = 0; 434 for (p = io_u->buf; p < io_u->buf + io_u->buflen; 435 p += hdr_inc, hdr_num++) { 436 if (ret && td->o.verify_fatal) { 437 td->terminate = 1; 438 break; 439 } 440 hdr_size = __hdr_size(td->o.verify); 441 if (td->o.verify_offset) 442 memswp(p, p + td->o.verify_offset, hdr_size); 443 hdr = p; 444 445 if (hdr->fio_magic != FIO_HDR_MAGIC) { 446 log_err("Bad verify header %x\n", hdr->fio_magic); 447 return EIO; 448 } 449 450 if (td->o.verify_pattern_bytes) { 451 dprint(FD_VERIFY, "pattern verify io_u %p, len %u\n", 452 io_u, hdr->len); 453 ret = verify_io_u_pattern(td->o.verify_pattern, 454 td->o.verify_pattern_bytes, 455 p + hdr_size, 456 hdr_inc - hdr_size, 457 hdr_size % 4); 458 if (ret) 459 log_err("fio: verify failed at %llu/%u\n", 460 io_u->offset + hdr_num * hdr->len, 461 hdr->len); 462 continue; 463 } 464 465 switch (hdr->verify_type) { 466 case VERIFY_MD5: 467 ret = verify_io_u_md5(hdr, io_u, hdr_num); 468 break; 469 case VERIFY_CRC64: 470 ret = verify_io_u_crc64(hdr, io_u, hdr_num); 471 break; 472 case VERIFY_CRC32C: 473 case VERIFY_CRC32C_INTEL: 474 ret = verify_io_u_crc32c(hdr, io_u, hdr_num); 475 break; 476 case VERIFY_CRC32: 477 ret = verify_io_u_crc32(hdr, io_u, hdr_num); 478 break; 479 case VERIFY_CRC16: 480 ret = verify_io_u_crc16(hdr, io_u, hdr_num); 481 break; 482 case VERIFY_CRC7: 483 ret = verify_io_u_crc7(hdr, io_u, hdr_num); 484 break; 485 case VERIFY_SHA256: 486 ret = verify_io_u_sha256(hdr, io_u, hdr_num); 487 break; 488 case VERIFY_SHA512: 489 ret = verify_io_u_sha512(hdr, io_u, hdr_num); 490 break; 491 case VERIFY_META: 492 ret = verify_io_u_meta(hdr, td, io_u, hdr_num); 493 break; 494 default: 495 log_err("Bad verify type %u\n", hdr->verify_type); 496 ret = EINVAL; 497 } 498 } 499 500 return ret; 501} 502 503static void fill_meta(struct verify_header *hdr, struct thread_data *td, 504 struct io_u *io_u, unsigned int header_num) 505{ 506 struct vhdr_meta *vh = hdr_priv(hdr); 507 508 vh->thread = td->thread_number; 509 510 vh->time_sec = io_u->start_time.tv_sec; 511 vh->time_usec = io_u->start_time.tv_usec; 512 513 vh->numberio = td->io_issues[DDIR_WRITE]; 514 515 vh->offset = io_u->offset + header_num * td->o.verify_interval; 516} 517 518static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len) 519{ 520 struct vhdr_sha512 *vh = hdr_priv(hdr); 521 struct sha512_ctx sha512_ctx = { 522 .buf = vh->sha512, 523 }; 524 525 sha512_init(&sha512_ctx); 526 sha512_update(&sha512_ctx, p, len); 527} 528 529static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len) 530{ 531 struct vhdr_sha256 *vh = hdr_priv(hdr); 532 struct sha256_ctx sha256_ctx = { 533 .buf = vh->sha256, 534 }; 535 536 sha256_init(&sha256_ctx); 537 sha256_update(&sha256_ctx, p, len); 538} 539 540static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len) 541{ 542 struct vhdr_crc7 *vh = hdr_priv(hdr); 543 544 vh->crc7 = crc7(p, len); 545} 546 547static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len) 548{ 549 struct vhdr_crc16 *vh = hdr_priv(hdr); 550 551 vh->crc16 = crc16(p, len); 552} 553 554static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len) 555{ 556 struct vhdr_crc32 *vh = hdr_priv(hdr); 557 558 vh->crc32 = crc32(p, len); 559} 560 561static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len) 562{ 563 struct vhdr_crc32 *vh = hdr_priv(hdr); 564 565 if (hdr->verify_type == VERIFY_CRC32C_INTEL) 566 vh->crc32 = crc32c_intel(p, len); 567 else 568 vh->crc32 = crc32c(p, len); 569} 570 571static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len) 572{ 573 struct vhdr_crc64 *vh = hdr_priv(hdr); 574 575 vh->crc64 = crc64(p, len); 576} 577 578static void fill_md5(struct verify_header *hdr, void *p, unsigned int len) 579{ 580 struct vhdr_md5 *vh = hdr_priv(hdr); 581 struct md5_ctx md5_ctx = { 582 .hash = (uint32_t *) vh->md5_digest, 583 }; 584 585 md5_init(&md5_ctx); 586 md5_update(&md5_ctx, p, len); 587} 588 589/* 590 * fill body of io_u->buf with random data and add a header with the 591 * crc32 or md5 sum of that data. 592 */ 593void populate_verify_io_u(struct thread_data *td, struct io_u *io_u) 594{ 595 struct verify_header *hdr; 596 void *p = io_u->buf, *data; 597 unsigned int hdr_inc, data_len, header_num = 0; 598 599 if (td->o.verify == VERIFY_NULL) 600 return; 601 602 fill_pattern(td, p, io_u->buflen); 603 604 hdr_inc = io_u->buflen; 605 if (td->o.verify_interval) 606 hdr_inc = td->o.verify_interval; 607 608 for (; p < io_u->buf + io_u->buflen; p += hdr_inc) { 609 hdr = p; 610 611 hdr->fio_magic = FIO_HDR_MAGIC; 612 hdr->verify_type = td->o.verify; 613 hdr->len = hdr_inc; 614 data_len = hdr_inc - hdr_size(hdr); 615 616 data = p + hdr_size(hdr); 617 switch (td->o.verify) { 618 case VERIFY_MD5: 619 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n", 620 io_u, hdr->len); 621 fill_md5(hdr, data, data_len); 622 break; 623 case VERIFY_CRC64: 624 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n", 625 io_u, hdr->len); 626 fill_crc64(hdr, data, data_len); 627 break; 628 case VERIFY_CRC32C: 629 case VERIFY_CRC32C_INTEL: 630 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n", 631 io_u, hdr->len); 632 fill_crc32c(hdr, data, data_len); 633 break; 634 case VERIFY_CRC32: 635 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n", 636 io_u, hdr->len); 637 fill_crc32(hdr, data, data_len); 638 break; 639 case VERIFY_CRC16: 640 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n", 641 io_u, hdr->len); 642 fill_crc16(hdr, data, data_len); 643 break; 644 case VERIFY_CRC7: 645 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n", 646 io_u, hdr->len); 647 fill_crc7(hdr, data, data_len); 648 break; 649 case VERIFY_SHA256: 650 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n", 651 io_u, hdr->len); 652 fill_sha256(hdr, data, data_len); 653 break; 654 case VERIFY_SHA512: 655 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n", 656 io_u, hdr->len); 657 fill_sha512(hdr, data, data_len); 658 break; 659 case VERIFY_META: 660 dprint(FD_VERIFY, "fill meta io_u %p, len %u\n", 661 io_u, hdr->len); 662 fill_meta(hdr, td, io_u, header_num); 663 break; 664 default: 665 log_err("fio: bad verify type: %d\n", td->o.verify); 666 assert(0); 667 } 668 if (td->o.verify_offset) 669 memswp(p, p + td->o.verify_offset, hdr_size(hdr)); 670 header_num++; 671 } 672} 673 674int get_next_verify(struct thread_data *td, struct io_u *io_u) 675{ 676 struct io_piece *ipo = NULL; 677 678 /* 679 * this io_u is from a requeue, we already filled the offsets 680 */ 681 if (io_u->file) 682 return 0; 683 684 if (!RB_EMPTY_ROOT(&td->io_hist_tree)) { 685 struct rb_node *n = rb_first(&td->io_hist_tree); 686 687 ipo = rb_entry(n, struct io_piece, rb_node); 688 rb_erase(n, &td->io_hist_tree); 689 } else if (!flist_empty(&td->io_hist_list)) { 690 ipo = flist_entry(td->io_hist_list.next, struct io_piece, list); 691 flist_del(&ipo->list); 692 } 693 694 if (ipo) { 695 io_u->offset = ipo->offset; 696 io_u->buflen = ipo->len; 697 io_u->file = ipo->file; 698 699 if ((io_u->file->flags & FIO_FILE_OPEN) == 0) { 700 int r = td_io_open_file(td, io_u->file); 701 702 if (r) { 703 dprint(FD_VERIFY, "failed file %s open\n", 704 io_u->file->file_name); 705 return 1; 706 } 707 } 708 709 get_file(ipo->file); 710 assert(io_u->file->flags & FIO_FILE_OPEN); 711 io_u->ddir = DDIR_READ; 712 io_u->xfer_buf = io_u->buf; 713 io_u->xfer_buflen = io_u->buflen; 714 free(ipo); 715 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u); 716 return 0; 717 } 718 719 dprint(FD_VERIFY, "get_next_verify: empty\n"); 720 return 1; 721} 722