verify.c revision a917a8b3dfeefdd7007ba2f46f21fc145574309d
1e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe/* 2e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe * IO verification helpers 3e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe */ 4e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe#include <unistd.h> 5e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe#include <fcntl.h> 6e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe#include <string.h> 797af62cec418cd722fb43c6010d6430534e12353Jens Axboe#include <assert.h> 8e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe#include <pthread.h> 9e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 10e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe#include "fio.h" 114f5af7b2370a6d3e64bc5128905c1aa8b0dc51b0Jens Axboe#include "verify.h" 12e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe#include "smalloc.h" 130d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe#include "trim.h" 14637ef8d9f7645135cf4829894d1e3983cd7a042eJens Axboe#include "lib/rand.h" 15e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 16eef6eea1b935a67f1ae26b38e06a69d4410a12aaJens Axboe#include "crc/md5.h" 17eef6eea1b935a67f1ae26b38e06a69d4410a12aaJens Axboe#include "crc/crc64.h" 18eef6eea1b935a67f1ae26b38e06a69d4410a12aaJens Axboe#include "crc/crc32.h" 19bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe#include "crc/crc32c.h" 20eef6eea1b935a67f1ae26b38e06a69d4410a12aaJens Axboe#include "crc/crc16.h" 21eef6eea1b935a67f1ae26b38e06a69d4410a12aaJens Axboe#include "crc/crc7.h" 22eef6eea1b935a67f1ae26b38e06a69d4410a12aaJens Axboe#include "crc/sha256.h" 23eef6eea1b935a67f1ae26b38e06a69d4410a12aaJens Axboe#include "crc/sha512.h" 247c353ceb3257b132f4c98326046b42201e070731Jens Axboe#include "crc/sha1.h" 25cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 26cbe8d7561cf6d81d741d87eb7940db2a111d2144Radha Ramachandranvoid fill_pattern(struct thread_data *td, void *p, unsigned int len, struct io_u *io_u) 2790059d65bde4804e4da07401ad6902c1d956b592Jens Axboe{ 2890059d65bde4804e4da07401ad6902c1d956b592Jens Axboe switch (td->o.verify_pattern_bytes) { 2990059d65bde4804e4da07401ad6902c1d956b592Jens Axboe case 0: 30bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "fill random bytes len=%u\n", len); 31637ef8d9f7645135cf4829894d1e3983cd7a042eJens Axboe fill_random_buf(p, len); 3290059d65bde4804e4da07401ad6902c1d956b592Jens Axboe break; 3390059d65bde4804e4da07401ad6902c1d956b592Jens Axboe case 1: 3410e8a7b30ce2917df9f839d42596c7e3af9a904fRadha Ramachandran if (io_u->buf_filled_len >= len) { 35cbe8d7561cf6d81d741d87eb7940db2a111d2144Radha Ramachandran dprint(FD_VERIFY, "using already filled verify pattern b=0 len=%u\n", len); 36cbe8d7561cf6d81d741d87eb7940db2a111d2144Radha Ramachandran return; 37cbe8d7561cf6d81d741d87eb7940db2a111d2144Radha Ramachandran } 38bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "fill verify pattern b=0 len=%u\n", len); 390e92f873a3d542d2b237d68fd9391c92a85fa0b8Radha Ramachandran memset(p, td->o.verify_pattern[0], len); 40cbe8d7561cf6d81d741d87eb7940db2a111d2144Radha Ramachandran io_u->buf_filled_len = len; 4190059d65bde4804e4da07401ad6902c1d956b592Jens Axboe break; 420e92f873a3d542d2b237d68fd9391c92a85fa0b8Radha Ramachandran default: { 430e92f873a3d542d2b237d68fd9391c92a85fa0b8Radha Ramachandran unsigned int i = 0, size = 0; 4490059d65bde4804e4da07401ad6902c1d956b592Jens Axboe unsigned char *b = p; 4590059d65bde4804e4da07401ad6902c1d956b592Jens Axboe 4610e8a7b30ce2917df9f839d42596c7e3af9a904fRadha Ramachandran if (io_u->buf_filled_len >= len) { 47cbe8d7561cf6d81d741d87eb7940db2a111d2144Radha Ramachandran dprint(FD_VERIFY, "using already filled verify pattern b=%d len=%u\n", 48cbe8d7561cf6d81d741d87eb7940db2a111d2144Radha Ramachandran td->o.verify_pattern_bytes, len); 49cbe8d7561cf6d81d741d87eb7940db2a111d2144Radha Ramachandran return; 50cbe8d7561cf6d81d741d87eb7940db2a111d2144Radha Ramachandran } 51bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "fill verify pattern b=%d len=%u\n", 52bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe td->o.verify_pattern_bytes, len); 53bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe 5490059d65bde4804e4da07401ad6902c1d956b592Jens Axboe while (i < len) { 550e92f873a3d542d2b237d68fd9391c92a85fa0b8Radha Ramachandran size = td->o.verify_pattern_bytes; 560e92f873a3d542d2b237d68fd9391c92a85fa0b8Radha Ramachandran if (size > (len - i)) 570e92f873a3d542d2b237d68fd9391c92a85fa0b8Radha Ramachandran size = len - i; 580e92f873a3d542d2b237d68fd9391c92a85fa0b8Radha Ramachandran memcpy(b+i, td->o.verify_pattern, size); 590e92f873a3d542d2b237d68fd9391c92a85fa0b8Radha Ramachandran i += size; 6090059d65bde4804e4da07401ad6902c1d956b592Jens Axboe } 61cbe8d7561cf6d81d741d87eb7940db2a111d2144Radha Ramachandran io_u->buf_filled_len = len; 6290059d65bde4804e4da07401ad6902c1d956b592Jens Axboe break; 6390059d65bde4804e4da07401ad6902c1d956b592Jens Axboe } 6490059d65bde4804e4da07401ad6902c1d956b592Jens Axboe } 6590059d65bde4804e4da07401ad6902c1d956b592Jens Axboe} 6690059d65bde4804e4da07401ad6902c1d956b592Jens Axboe 674764aec9fe5988281bdc114c13ac771fb16713ccJens Axboestatic void memswp(void *buf1, void *buf2, unsigned int len) 68546a9142511875524850ac92776184fd9fb7196eShawn Lewis{ 69dee6de74152299f2a7df07ae971a53441b971dafShawn Lewis char swap[200]; 70dee6de74152299f2a7df07ae971a53441b971dafShawn Lewis 71dee6de74152299f2a7df07ae971a53441b971dafShawn Lewis assert(len <= sizeof(swap)); 7290059d65bde4804e4da07401ad6902c1d956b592Jens Axboe 73546a9142511875524850ac92776184fd9fb7196eShawn Lewis memcpy(&swap, buf1, len); 74546a9142511875524850ac92776184fd9fb7196eShawn Lewis memcpy(buf1, buf2, len); 75546a9142511875524850ac92776184fd9fb7196eShawn Lewis memcpy(buf2, &swap, len); 76546a9142511875524850ac92776184fd9fb7196eShawn Lewis} 77546a9142511875524850ac92776184fd9fb7196eShawn Lewis 78e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboestatic void hexdump(void *buffer, int len) 79e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe{ 80e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe unsigned char *p = buffer; 81e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe int i; 82e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 83e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe for (i = 0; i < len; i++) 84bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_err("%02x", p[i]); 85bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_err("\n"); 86e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe} 87e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 88d9f2caf3de3f2ba6f8ecd8a2688e1eb7715bc2aaJens Axboe/* 898767783245faac99a7e1c330a1d6536e579c8402Jens Axboe * Prepare for seperation of verify_header and checksum header 908767783245faac99a7e1c330a1d6536e579c8402Jens Axboe */ 91546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboestatic inline unsigned int __hdr_size(int verify_type) 928767783245faac99a7e1c330a1d6536e579c8402Jens Axboe{ 935921e80c5dfc9f96d2f21da6ae58f2b5d3a0b373Jens Axboe unsigned int len = len; 948767783245faac99a7e1c330a1d6536e579c8402Jens Axboe 95546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe switch (verify_type) { 96546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe case VERIFY_NONE: 97546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe case VERIFY_NULL: 98546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe len = 0; 99546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe break; 100546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe case VERIFY_MD5: 101546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe len = sizeof(struct vhdr_md5); 102546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe break; 103546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe case VERIFY_CRC64: 104546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe len = sizeof(struct vhdr_crc64); 105546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe break; 106bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe case VERIFY_CRC32C: 107546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe case VERIFY_CRC32: 1083845591fadea480177223e28c9d1c03642d34f0eJens Axboe case VERIFY_CRC32C_INTEL: 109546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe len = sizeof(struct vhdr_crc32); 110546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe break; 111546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe case VERIFY_CRC16: 112546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe len = sizeof(struct vhdr_crc16); 113546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe break; 114546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe case VERIFY_CRC7: 115546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe len = sizeof(struct vhdr_crc7); 116546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe break; 117546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe case VERIFY_SHA256: 118546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe len = sizeof(struct vhdr_sha256); 119546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe break; 120546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe case VERIFY_SHA512: 121546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe len = sizeof(struct vhdr_sha512); 122546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe break; 1237437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis case VERIFY_META: 1247437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis len = sizeof(struct vhdr_meta); 1257437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis break; 1267c353ceb3257b132f4c98326046b42201e070731Jens Axboe case VERIFY_SHA1: 1277c353ceb3257b132f4c98326046b42201e070731Jens Axboe len = sizeof(struct vhdr_sha1); 1287c353ceb3257b132f4c98326046b42201e070731Jens Axboe break; 129546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe default: 130546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe log_err("fio: unknown verify header!\n"); 131546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe assert(0); 132546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe } 133546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe 134546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe return len + sizeof(struct verify_header); 1358767783245faac99a7e1c330a1d6536e579c8402Jens Axboe} 1368767783245faac99a7e1c330a1d6536e579c8402Jens Axboe 1378767783245faac99a7e1c330a1d6536e579c8402Jens Axboestatic inline unsigned int hdr_size(struct verify_header *hdr) 1388767783245faac99a7e1c330a1d6536e579c8402Jens Axboe{ 139546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe return __hdr_size(hdr->verify_type); 140546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe} 141546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe 142546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboestatic void *hdr_priv(struct verify_header *hdr) 143546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe{ 144546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe void *priv = hdr; 145546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe 146546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe return priv + sizeof(struct verify_header); 1478767783245faac99a7e1c330a1d6536e579c8402Jens Axboe} 1488767783245faac99a7e1c330a1d6536e579c8402Jens Axboe 1498767783245faac99a7e1c330a1d6536e579c8402Jens Axboe/* 150936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe * Verify container, pass info to verify handlers and allow them to 151936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe * pass info back in case of error 152936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe */ 153936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestruct vcont { 154936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe /* 155936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe * Input 156936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe */ 157936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe struct io_u *io_u; 158936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe unsigned int hdr_num; 159936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe 160936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe /* 161936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe * Output, only valid in case of error 162936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe */ 163bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe const char *name; 164bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe void *good_crc; 165bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe void *bad_crc; 166936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe unsigned int crc_len; 167936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe}; 168936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe 169bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboestatic void log_verify_failure(struct verify_header *hdr, struct vcont *vc) 170bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe{ 171bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe unsigned long long offset; 172bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe 173bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe offset = vc->io_u->offset; 174bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe offset += vc->hdr_num * hdr->len; 17532c17adf7f75da1f0957d4691633fea60259910fJens Axboe log_err("%.8s: verify failed at file %s offset %llu, length %u\n", 17632c17adf7f75da1f0957d4691633fea60259910fJens Axboe vc->name, vc->io_u->file->file_name, offset, hdr->len); 177bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe 178bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (vc->good_crc && vc->bad_crc) { 179bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_err(" Expected CRC: "); 180bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe hexdump(vc->good_crc, vc->crc_len); 181bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_err(" Received CRC: "); 182bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe hexdump(vc->bad_crc, vc->crc_len); 183bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe } 184bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe} 185bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe 186936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe/* 187d9f2caf3de3f2ba6f8ecd8a2688e1eb7715bc2aaJens Axboe * Return data area 'header_num' 188d9f2caf3de3f2ba6f8ecd8a2688e1eb7715bc2aaJens Axboe */ 189936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestatic inline void *io_u_verify_off(struct verify_header *hdr, struct vcont *vc) 190d9f2caf3de3f2ba6f8ecd8a2688e1eb7715bc2aaJens Axboe{ 191936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe return vc->io_u->buf + vc->hdr_num * hdr->len + hdr_size(hdr); 192d9f2caf3de3f2ba6f8ecd8a2688e1eb7715bc2aaJens Axboe} 193d9f2caf3de3f2ba6f8ecd8a2688e1eb7715bc2aaJens Axboe 1947437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewisstatic int verify_io_u_meta(struct verify_header *hdr, struct thread_data *td, 195936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe struct vcont *vc) 1967437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis{ 1977437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis struct vhdr_meta *vh = hdr_priv(hdr); 198936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe struct io_u *io_u = vc->io_u; 1997437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis 200bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len); 201bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe 202bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (vh->offset == io_u->offset + vc->hdr_num * td->o.verify_interval) 203bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return 0; 2047437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis 205bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->name = "meta"; 206bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_verify_failure(hdr, vc); 207bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return EILSEQ; 2087437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis} 2097437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis 210936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestatic int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc) 211cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe{ 212936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe void *p = io_u_verify_off(hdr, vc); 213546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_sha512 *vh = hdr_priv(hdr); 214cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe uint8_t sha512[128]; 215cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe struct sha512_ctx sha512_ctx = { 216cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe .buf = sha512, 217cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe }; 218cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 219936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", vc->io_u, hdr->len); 220bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe 221cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe sha512_init(&sha512_ctx); 2228767783245faac99a7e1c330a1d6536e579c8402Jens Axboe sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr)); 223cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 224bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (!memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512))) 225bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return 0; 226cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 227bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->name = "sha512"; 228bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->good_crc = vh->sha512; 229bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->bad_crc = sha512_ctx.buf; 230bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->crc_len = sizeof(vh->sha512); 231bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_verify_failure(hdr, vc); 232bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return EILSEQ; 233cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe} 234cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 235936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestatic int verify_io_u_sha256(struct verify_header *hdr, struct vcont *vc) 236cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe{ 237936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe void *p = io_u_verify_off(hdr, vc); 238546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_sha256 *vh = hdr_priv(hdr); 239bc77f56f79f6ae155e9eb26263ed686ef28b47ecJens Axboe uint8_t sha256[64]; 240cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe struct sha256_ctx sha256_ctx = { 241cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe .buf = sha256, 242cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe }; 243cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 244936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", vc->io_u, hdr->len); 245bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe 246cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe sha256_init(&sha256_ctx); 2478767783245faac99a7e1c330a1d6536e579c8402Jens Axboe sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr)); 248cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 249bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (!memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256))) 250bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return 0; 251cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 252bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->name = "sha256"; 253bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->good_crc = vh->sha256; 254bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->bad_crc = sha256_ctx.buf; 255bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->crc_len = sizeof(vh->sha256); 256bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_verify_failure(hdr, vc); 257bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return EILSEQ; 258cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe} 259cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 260936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestatic int verify_io_u_sha1(struct verify_header *hdr, struct vcont *vc) 2617c353ceb3257b132f4c98326046b42201e070731Jens Axboe{ 262936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe void *p = io_u_verify_off(hdr, vc); 2637c353ceb3257b132f4c98326046b42201e070731Jens Axboe struct vhdr_sha1 *vh = hdr_priv(hdr); 2647c353ceb3257b132f4c98326046b42201e070731Jens Axboe uint32_t sha1[5]; 2657c353ceb3257b132f4c98326046b42201e070731Jens Axboe struct sha1_ctx sha1_ctx = { 2667c353ceb3257b132f4c98326046b42201e070731Jens Axboe .H = sha1, 2677c353ceb3257b132f4c98326046b42201e070731Jens Axboe }; 2687c353ceb3257b132f4c98326046b42201e070731Jens Axboe 269936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", vc->io_u, hdr->len); 2707c353ceb3257b132f4c98326046b42201e070731Jens Axboe 2717c353ceb3257b132f4c98326046b42201e070731Jens Axboe sha1_init(&sha1_ctx); 2727c353ceb3257b132f4c98326046b42201e070731Jens Axboe sha1_update(&sha1_ctx, p, hdr->len - hdr_size(hdr)); 2737c353ceb3257b132f4c98326046b42201e070731Jens Axboe 274bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (!memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1))) 275bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return 0; 2767c353ceb3257b132f4c98326046b42201e070731Jens Axboe 277bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->name = "sha1"; 278bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->good_crc = vh->sha1; 279bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->bad_crc = sha1_ctx.H; 280bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->crc_len = sizeof(vh->sha1); 281bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_verify_failure(hdr, vc); 282bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return EILSEQ; 2837c353ceb3257b132f4c98326046b42201e070731Jens Axboe} 2847c353ceb3257b132f4c98326046b42201e070731Jens Axboe 285936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestatic int verify_io_u_crc7(struct verify_header *hdr, struct vcont *vc) 2861e154bdbf14a1d352117aea057035235b66f0381Jens Axboe{ 287936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe void *p = io_u_verify_off(hdr, vc); 288546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_crc7 *vh = hdr_priv(hdr); 2891e154bdbf14a1d352117aea057035235b66f0381Jens Axboe unsigned char c; 2901e154bdbf14a1d352117aea057035235b66f0381Jens Axboe 291936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", vc->io_u, hdr->len); 292bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe 2938767783245faac99a7e1c330a1d6536e579c8402Jens Axboe c = crc7(p, hdr->len - hdr_size(hdr)); 2941e154bdbf14a1d352117aea057035235b66f0381Jens Axboe 295bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (c == vh->crc7) 296bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return 0; 2971e154bdbf14a1d352117aea057035235b66f0381Jens Axboe 298bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->name = "crc7"; 299bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->good_crc = &vh->crc7; 300bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->bad_crc = &c; 301bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->crc_len = 1; 302bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_verify_failure(hdr, vc); 303bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return EILSEQ; 3041e154bdbf14a1d352117aea057035235b66f0381Jens Axboe} 3051e154bdbf14a1d352117aea057035235b66f0381Jens Axboe 306936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestatic int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc) 307969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboe{ 308936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe void *p = io_u_verify_off(hdr, vc); 309546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_crc16 *vh = hdr_priv(hdr); 310969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboe unsigned short c; 311969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboe 312936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len); 313bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe 3148767783245faac99a7e1c330a1d6536e579c8402Jens Axboe c = crc16(p, hdr->len - hdr_size(hdr)); 315969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboe 316bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (c == vh->crc16) 317bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return 0; 318969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboe 319bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->name = "crc16"; 320bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->good_crc = &vh->crc16; 321bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->bad_crc = &c; 322bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->crc_len = 2; 323bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_verify_failure(hdr, vc); 324bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return EILSEQ; 325969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboe} 326969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboe 327936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestatic int verify_io_u_crc64(struct verify_header *hdr, struct vcont *vc) 328d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboe{ 329936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe void *p = io_u_verify_off(hdr, vc); 330546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_crc64 *vh = hdr_priv(hdr); 331d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboe unsigned long long c; 332d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboe 333936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", vc->io_u, hdr->len); 334bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe 3358767783245faac99a7e1c330a1d6536e579c8402Jens Axboe c = crc64(p, hdr->len - hdr_size(hdr)); 336d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboe 337bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (c == vh->crc64) 338bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return 0; 339d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboe 340bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->name = "crc64"; 341bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->good_crc = &vh->crc64; 342bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->bad_crc = &c; 343bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->crc_len = 8; 344bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_verify_failure(hdr, vc); 345bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return EILSEQ; 346d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboe} 347d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboe 348936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestatic int verify_io_u_crc32(struct verify_header *hdr, struct vcont *vc) 349e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe{ 350936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe void *p = io_u_verify_off(hdr, vc); 351546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_crc32 *vh = hdr_priv(hdr); 352546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe uint32_t c; 353e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 354936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", vc->io_u, hdr->len); 355bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe 3568767783245faac99a7e1c330a1d6536e579c8402Jens Axboe c = crc32(p, hdr->len - hdr_size(hdr)); 357e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 358bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (c == vh->crc32) 359bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return 0; 360e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 361bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->name = "crc32"; 362bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->good_crc = &vh->crc32; 363bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->bad_crc = &c; 364bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->crc_len = 4; 365bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_verify_failure(hdr, vc); 366bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return EILSEQ; 367e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe} 368e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 369936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestatic int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc) 370bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe{ 371936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe void *p = io_u_verify_off(hdr, vc); 372bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe struct vhdr_crc32 *vh = hdr_priv(hdr); 373bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe uint32_t c; 374bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe 375936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len); 376bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe 3773845591fadea480177223e28c9d1c03642d34f0eJens Axboe if (hdr->verify_type == VERIFY_CRC32C_INTEL) 3783845591fadea480177223e28c9d1c03642d34f0eJens Axboe c = crc32c_intel(p, hdr->len - hdr_size(hdr)); 3793845591fadea480177223e28c9d1c03642d34f0eJens Axboe else 3803845591fadea480177223e28c9d1c03642d34f0eJens Axboe c = crc32c(p, hdr->len - hdr_size(hdr)); 381bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe 382bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (c == vh->crc32) 383bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return 0; 384bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe 385bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->name = "crc32c"; 386bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->good_crc = &vh->crc32; 387bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->bad_crc = &c; 388bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->crc_len = 4; 389bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_verify_failure(hdr, vc); 390bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return EILSEQ; 391bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe} 392bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe 393936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestatic int verify_io_u_md5(struct verify_header *hdr, struct vcont *vc) 394e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe{ 395936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe void *p = io_u_verify_off(hdr, vc); 396546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_md5 *vh = hdr_priv(hdr); 3978c432325c3df2075a77b27eab8a87704cf7b48eeJens Axboe uint32_t hash[MD5_HASH_WORDS]; 3988c432325c3df2075a77b27eab8a87704cf7b48eeJens Axboe struct md5_ctx md5_ctx = { 3998c432325c3df2075a77b27eab8a87704cf7b48eeJens Axboe .hash = hash, 4008c432325c3df2075a77b27eab8a87704cf7b48eeJens Axboe }; 401e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 402936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", vc->io_u, hdr->len); 403bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe 40461f821f16cf27d8742e09b3a137c933cd0085e9bJens Axboe md5_init(&md5_ctx); 4058767783245faac99a7e1c330a1d6536e579c8402Jens Axboe md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr)); 406e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 407bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (!memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash))) 408bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return 0; 409e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 410bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->name = "md5"; 411bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->good_crc = vh->md5_digest; 412bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->bad_crc = md5_ctx.hash; 413bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->crc_len = sizeof(hash); 414bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_verify_failure(hdr, vc); 415bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return EILSEQ; 416e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe} 417e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 4183f199b01fc9cc7a0f4915db266d7a28bc46e84f0Jens Axboestatic unsigned int hweight8(unsigned int w) 4193f199b01fc9cc7a0f4915db266d7a28bc46e84f0Jens Axboe{ 4203f199b01fc9cc7a0f4915db266d7a28bc46e84f0Jens Axboe unsigned int res = w - ((w >> 1) & 0x55); 4213f199b01fc9cc7a0f4915db266d7a28bc46e84f0Jens Axboe 4223f199b01fc9cc7a0f4915db266d7a28bc46e84f0Jens Axboe res = (res & 0x33) + ((res >> 2) & 0x33); 4233f199b01fc9cc7a0f4915db266d7a28bc46e84f0Jens Axboe return (res + (res >> 4)) & 0x0F; 4243f199b01fc9cc7a0f4915db266d7a28bc46e84f0Jens Axboe} 4253f199b01fc9cc7a0f4915db266d7a28bc46e84f0Jens Axboe 4260e92f873a3d542d2b237d68fd9391c92a85fa0b8Radha Ramachandranint verify_io_u_pattern(char *pattern, unsigned long pattern_size, 4275ec10eaad3b09875b91e19a20bbdfa06f2117562Jens Axboe char *buf, unsigned int len, unsigned int mod) 428a944e335fac62c75966b417a25ec107c40957cdbShawn Lewis{ 429a944e335fac62c75966b417a25ec107c40957cdbShawn Lewis unsigned int i; 430a944e335fac62c75966b417a25ec107c40957cdbShawn Lewis 431a944e335fac62c75966b417a25ec107c40957cdbShawn Lewis for (i = 0; i < len; i++) { 4320e92f873a3d542d2b237d68fd9391c92a85fa0b8Radha Ramachandran if (buf[i] != pattern[mod]) { 4333f199b01fc9cc7a0f4915db266d7a28bc46e84f0Jens Axboe unsigned int bits; 4343f199b01fc9cc7a0f4915db266d7a28bc46e84f0Jens Axboe 4350e92f873a3d542d2b237d68fd9391c92a85fa0b8Radha Ramachandran bits = hweight8(buf[i] ^ pattern[mod]); 4363f199b01fc9cc7a0f4915db266d7a28bc46e84f0Jens Axboe log_err("fio: got pattern %x, wanted %x. Bad bits %d\n", 4370e92f873a3d542d2b237d68fd9391c92a85fa0b8Radha Ramachandran buf[i], pattern[mod], bits); 4383f199b01fc9cc7a0f4915db266d7a28bc46e84f0Jens Axboe log_err("fio: bad pattern block offset %u\n", i); 4399fd189697dbf9fb7878093be5e41f25b7f932c2dJens Axboe return EILSEQ; 4403f199b01fc9cc7a0f4915db266d7a28bc46e84f0Jens Axboe } 441a944e335fac62c75966b417a25ec107c40957cdbShawn Lewis mod++; 442a944e335fac62c75966b417a25ec107c40957cdbShawn Lewis if (mod == pattern_size) 443a944e335fac62c75966b417a25ec107c40957cdbShawn Lewis mod = 0; 444a944e335fac62c75966b417a25ec107c40957cdbShawn Lewis } 445a944e335fac62c75966b417a25ec107c40957cdbShawn Lewis 446a944e335fac62c75966b417a25ec107c40957cdbShawn Lewis return 0; 447a944e335fac62c75966b417a25ec107c40957cdbShawn Lewis} 448a944e335fac62c75966b417a25ec107c40957cdbShawn Lewis 449e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe/* 450e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe * Push IO verification to a separate thread 451e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe */ 452e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboeint verify_io_u_async(struct thread_data *td, struct io_u *io_u) 453e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe{ 454e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe if (io_u->file) 455e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe put_file_log(td, io_u->file); 456e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 457e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe io_u->file = NULL; 458e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 459e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_mutex_lock(&td->io_u_lock); 4600c41214ff4e6f31f8df64aac37be7853aada6f1fRadha Ramachandran 4610c41214ff4e6f31f8df64aac37be7853aada6f1fRadha Ramachandran if (io_u->flags & IO_U_F_IN_CUR_DEPTH) { 4620c41214ff4e6f31f8df64aac37be7853aada6f1fRadha Ramachandran td->cur_depth--; 4630c41214ff4e6f31f8df64aac37be7853aada6f1fRadha Ramachandran io_u->flags &= ~IO_U_F_IN_CUR_DEPTH; 4640c41214ff4e6f31f8df64aac37be7853aada6f1fRadha Ramachandran } 465e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe flist_del(&io_u->list); 466e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe flist_add_tail(&io_u->list, &td->verify_list); 4672ecc1b57721e3cb72bbf558bc169c97037fe3d0aJens Axboe io_u->flags |= IO_U_F_FREE_DEF; 468e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_mutex_unlock(&td->io_u_lock); 469e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 470e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_cond_signal(&td->verify_cond); 471e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe return 0; 472e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe} 473e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 4740d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboestatic int verify_trimmed_io_u(struct thread_data *td, struct io_u *io_u) 4750d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe{ 4760d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe static char zero_buf[1024]; 4770d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe unsigned int this_len, len; 4780d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe int ret = 0; 4790d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe void *p; 4800d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe 4810d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe if (!td->o.trim_zero) 4820d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe return 0; 4830d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe 4840d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe len = io_u->buflen; 4850d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe p = io_u->buf; 4860d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe do { 4870d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe this_len = sizeof(zero_buf); 4880d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe if (this_len > len) 4890d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe this_len = len; 4900d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe if (memcmp(p, zero_buf, this_len)) { 4910d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe ret = EILSEQ; 4920d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe break; 4930d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe } 4940d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe len -= this_len; 4950d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe p += this_len; 4960d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe } while (len); 4970d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe 4980d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe if (!ret) 4990d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe return 0; 5000d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe 501a917a8b3dfeefdd7007ba2f46f21fc145574309dJens Axboe log_err("trim: verify failed at file %s offset %llu, length %lu" 502a917a8b3dfeefdd7007ba2f46f21fc145574309dJens Axboe ", block offset %lu\n", 503a917a8b3dfeefdd7007ba2f46f21fc145574309dJens Axboe io_u->file->file_name, io_u->offset, io_u->buflen, 504a917a8b3dfeefdd7007ba2f46f21fc145574309dJens Axboe (p - io_u->buf)); 5050d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe return ret; 5060d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe} 5070d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe 50836690c9b5b50736cec1fcd53100f01fd1181fdf4Jens Axboeint verify_io_u(struct thread_data *td, struct io_u *io_u) 509e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe{ 5103f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis struct verify_header *hdr; 511a944e335fac62c75966b417a25ec107c40957cdbShawn Lewis unsigned int hdr_size, hdr_inc, hdr_num = 0; 51295646108f83a34c0486d7f3ed19e99c1d93691e6Jens Axboe void *p; 513e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe int ret; 514e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 5151dcc049824b840df1ee0ef5d019fffa4037e1bb0Shawn Lewis if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ) 51636690c9b5b50736cec1fcd53100f01fd1181fdf4Jens Axboe return 0; 5170d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe if (io_u->flags & IO_U_F_TRIMMED) { 5180d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe ret = verify_trimmed_io_u(td, io_u); 5190d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe goto done; 5200d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe } 52136690c9b5b50736cec1fcd53100f01fd1181fdf4Jens Axboe 5223f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis hdr_inc = io_u->buflen; 523a59e170d683f9ccbaa45648d7ae9a13b70e3822cJens Axboe if (td->o.verify_interval) 524a59e170d683f9ccbaa45648d7ae9a13b70e3822cJens Axboe hdr_inc = td->o.verify_interval; 525e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 526a12a3b4d8903b9941705067442c2c530a94635f3Jens Axboe ret = 0; 5275ec10eaad3b09875b91e19a20bbdfa06f2117562Jens Axboe for (p = io_u->buf; p < io_u->buf + io_u->buflen; 5285ec10eaad3b09875b91e19a20bbdfa06f2117562Jens Axboe p += hdr_inc, hdr_num++) { 529bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe struct vcont vc = { 530bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe .io_u = io_u, 531bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe .hdr_num = hdr_num, 532bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe }; 533936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe 534f3e6cb95fecf867ec46e3dfd59ef1a1db1b6ad2aJens Axboe if (ret && td->o.verify_fatal) 535a12a3b4d8903b9941705067442c2c530a94635f3Jens Axboe break; 536f3e6cb95fecf867ec46e3dfd59ef1a1db1b6ad2aJens Axboe 537a944e335fac62c75966b417a25ec107c40957cdbShawn Lewis hdr_size = __hdr_size(td->o.verify); 538a59e170d683f9ccbaa45648d7ae9a13b70e3822cJens Axboe if (td->o.verify_offset) 539a944e335fac62c75966b417a25ec107c40957cdbShawn Lewis memswp(p, p + td->o.verify_offset, hdr_size); 54095646108f83a34c0486d7f3ed19e99c1d93691e6Jens Axboe hdr = p; 541e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 5423f199b01fc9cc7a0f4915db266d7a28bc46e84f0Jens Axboe if (hdr->fio_magic != FIO_HDR_MAGIC) { 543c9b440312ff63c2b4c8548530f24c2eb5820b6a3Jens Axboe log_err("verify: bad magic header %x, wanted %x at file %s offset %llu, length %u\n", 544c9b440312ff63c2b4c8548530f24c2eb5820b6a3Jens Axboe hdr->fio_magic, FIO_HDR_MAGIC, 545c9b440312ff63c2b4c8548530f24c2eb5820b6a3Jens Axboe io_u->file->file_name, 546c9b440312ff63c2b4c8548530f24c2eb5820b6a3Jens Axboe io_u->offset + hdr_num * hdr->len, hdr->len); 5479fd189697dbf9fb7878093be5e41f25b7f932c2dJens Axboe return EILSEQ; 5483f199b01fc9cc7a0f4915db266d7a28bc46e84f0Jens Axboe } 5493f199b01fc9cc7a0f4915db266d7a28bc46e84f0Jens Axboe 550e28218f3017e5eb156d3a4a6e7b33651dc022ceaShawn Lewis if (td->o.verify_pattern_bytes) { 551bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "pattern verify io_u %p, len %u\n", 552bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe io_u, hdr->len); 553e28218f3017e5eb156d3a4a6e7b33651dc022ceaShawn Lewis ret = verify_io_u_pattern(td->o.verify_pattern, 5540e92f873a3d542d2b237d68fd9391c92a85fa0b8Radha Ramachandran td->o.verify_pattern_bytes, 5550e92f873a3d542d2b237d68fd9391c92a85fa0b8Radha Ramachandran p + hdr_size, 5560e92f873a3d542d2b237d68fd9391c92a85fa0b8Radha Ramachandran hdr_inc - hdr_size, 5570e92f873a3d542d2b237d68fd9391c92a85fa0b8Radha Ramachandran hdr_size % td->o.verify_pattern_bytes); 558c9b440312ff63c2b4c8548530f24c2eb5820b6a3Jens Axboe 559c9b440312ff63c2b4c8548530f24c2eb5820b6a3Jens Axboe if (ret) { 560c9b440312ff63c2b4c8548530f24c2eb5820b6a3Jens Axboe log_err("pattern: verify failed at file %s offset %llu, length %u\n", 561c9b440312ff63c2b4c8548530f24c2eb5820b6a3Jens Axboe io_u->file->file_name, 562c9b440312ff63c2b4c8548530f24c2eb5820b6a3Jens Axboe io_u->offset + hdr_num * hdr->len, 563c9b440312ff63c2b4c8548530f24c2eb5820b6a3Jens Axboe hdr->len); 564c9b440312ff63c2b4c8548530f24c2eb5820b6a3Jens Axboe } 565c9b440312ff63c2b4c8548530f24c2eb5820b6a3Jens Axboe 566e92d3d714acfe180413f63a5e5327de594aad0efRadha Ramachandran /* 567e92d3d714acfe180413f63a5e5327de594aad0efRadha Ramachandran * Also verify the meta data, if applicable 568e92d3d714acfe180413f63a5e5327de594aad0efRadha Ramachandran */ 569e92d3d714acfe180413f63a5e5327de594aad0efRadha Ramachandran if (hdr->verify_type == VERIFY_META) 570936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe ret |= verify_io_u_meta(hdr, td, &vc); 571e28218f3017e5eb156d3a4a6e7b33651dc022ceaShawn Lewis continue; 572e28218f3017e5eb156d3a4a6e7b33651dc022ceaShawn Lewis } 573e28218f3017e5eb156d3a4a6e7b33651dc022ceaShawn Lewis 5743f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis switch (hdr->verify_type) { 5753f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis case VERIFY_MD5: 576936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe ret = verify_io_u_md5(hdr, &vc); 5773f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis break; 5783f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis case VERIFY_CRC64: 579936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe ret = verify_io_u_crc64(hdr, &vc); 5803f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis break; 581bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe case VERIFY_CRC32C: 5823845591fadea480177223e28c9d1c03642d34f0eJens Axboe case VERIFY_CRC32C_INTEL: 583936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe ret = verify_io_u_crc32c(hdr, &vc); 584bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe break; 5853f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis case VERIFY_CRC32: 586936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe ret = verify_io_u_crc32(hdr, &vc); 5873f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis break; 5883f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis case VERIFY_CRC16: 589936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe ret = verify_io_u_crc16(hdr, &vc); 5903f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis break; 5913f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis case VERIFY_CRC7: 592936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe ret = verify_io_u_crc7(hdr, &vc); 5933f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis break; 594cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe case VERIFY_SHA256: 595936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe ret = verify_io_u_sha256(hdr, &vc); 596cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe break; 597cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe case VERIFY_SHA512: 598936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe ret = verify_io_u_sha512(hdr, &vc); 599cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe break; 6007437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis case VERIFY_META: 601936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe ret = verify_io_u_meta(hdr, td, &vc); 6027437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis break; 6037c353ceb3257b132f4c98326046b42201e070731Jens Axboe case VERIFY_SHA1: 604936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe ret = verify_io_u_sha1(hdr, &vc); 6057c353ceb3257b132f4c98326046b42201e070731Jens Axboe break; 6063f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis default: 6073f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis log_err("Bad verify type %u\n", hdr->verify_type); 608d16d4e09352f618c266d1efe27cbb25a95df6e86Jens Axboe ret = EINVAL; 6093f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis } 6103f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis } 611a7dfe862c2e3fe72b8db6a6e67302048d3731c6dJens Axboe 6120d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboedone: 613f3e6cb95fecf867ec46e3dfd59ef1a1db1b6ad2aJens Axboe if (ret && td->o.verify_fatal) 614f3e6cb95fecf867ec46e3dfd59ef1a1db1b6ad2aJens Axboe td->terminate = 1; 615f3e6cb95fecf867ec46e3dfd59ef1a1db1b6ad2aJens Axboe 616a12a3b4d8903b9941705067442c2c530a94635f3Jens Axboe return ret; 617e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe} 618e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 6197437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewisstatic void fill_meta(struct verify_header *hdr, struct thread_data *td, 6205ec10eaad3b09875b91e19a20bbdfa06f2117562Jens Axboe struct io_u *io_u, unsigned int header_num) 6217437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis{ 6227437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis struct vhdr_meta *vh = hdr_priv(hdr); 6237437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis 6247437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis vh->thread = td->thread_number; 6257437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis 6267437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis vh->time_sec = io_u->start_time.tv_sec; 6277437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis vh->time_usec = io_u->start_time.tv_usec; 6287437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis 6297437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis vh->numberio = td->io_issues[DDIR_WRITE]; 6307437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis 6317437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis vh->offset = io_u->offset + header_num * td->o.verify_interval; 6327437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis} 6337437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis 634cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboestatic void fill_sha512(struct verify_header *hdr, void *p, unsigned int len) 635cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe{ 636546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_sha512 *vh = hdr_priv(hdr); 637cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe struct sha512_ctx sha512_ctx = { 638546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe .buf = vh->sha512, 639cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe }; 640cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 641cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe sha512_init(&sha512_ctx); 642cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe sha512_update(&sha512_ctx, p, len); 643cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe} 644cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 645cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboestatic void fill_sha256(struct verify_header *hdr, void *p, unsigned int len) 646cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe{ 647546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_sha256 *vh = hdr_priv(hdr); 648cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe struct sha256_ctx sha256_ctx = { 649546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe .buf = vh->sha256, 650cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe }; 651cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 652cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe sha256_init(&sha256_ctx); 653cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe sha256_update(&sha256_ctx, p, len); 654cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe} 655cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 6567c353ceb3257b132f4c98326046b42201e070731Jens Axboestatic void fill_sha1(struct verify_header *hdr, void *p, unsigned int len) 6577c353ceb3257b132f4c98326046b42201e070731Jens Axboe{ 6587c353ceb3257b132f4c98326046b42201e070731Jens Axboe struct vhdr_sha1 *vh = hdr_priv(hdr); 6597c353ceb3257b132f4c98326046b42201e070731Jens Axboe struct sha1_ctx sha1_ctx = { 6607c353ceb3257b132f4c98326046b42201e070731Jens Axboe .H = vh->sha1, 6617c353ceb3257b132f4c98326046b42201e070731Jens Axboe }; 6627c353ceb3257b132f4c98326046b42201e070731Jens Axboe 6637c353ceb3257b132f4c98326046b42201e070731Jens Axboe sha1_init(&sha1_ctx); 6647c353ceb3257b132f4c98326046b42201e070731Jens Axboe sha1_update(&sha1_ctx, p, len); 6657c353ceb3257b132f4c98326046b42201e070731Jens Axboe} 6667c353ceb3257b132f4c98326046b42201e070731Jens Axboe 6671e154bdbf14a1d352117aea057035235b66f0381Jens Axboestatic void fill_crc7(struct verify_header *hdr, void *p, unsigned int len) 6681e154bdbf14a1d352117aea057035235b66f0381Jens Axboe{ 669546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_crc7 *vh = hdr_priv(hdr); 670546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe 671546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe vh->crc7 = crc7(p, len); 6721e154bdbf14a1d352117aea057035235b66f0381Jens Axboe} 6731e154bdbf14a1d352117aea057035235b66f0381Jens Axboe 674969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboestatic void fill_crc16(struct verify_header *hdr, void *p, unsigned int len) 675969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboe{ 676546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_crc16 *vh = hdr_priv(hdr); 677546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe 678546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe vh->crc16 = crc16(p, len); 679969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboe} 680969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboe 681e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboestatic void fill_crc32(struct verify_header *hdr, void *p, unsigned int len) 682e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe{ 683546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_crc32 *vh = hdr_priv(hdr); 684546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe 685546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe vh->crc32 = crc32(p, len); 686e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe} 687e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 688bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboestatic void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len) 689bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe{ 690bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe struct vhdr_crc32 *vh = hdr_priv(hdr); 691bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe 6923845591fadea480177223e28c9d1c03642d34f0eJens Axboe if (hdr->verify_type == VERIFY_CRC32C_INTEL) 6933845591fadea480177223e28c9d1c03642d34f0eJens Axboe vh->crc32 = crc32c_intel(p, len); 6943845591fadea480177223e28c9d1c03642d34f0eJens Axboe else 6953845591fadea480177223e28c9d1c03642d34f0eJens Axboe vh->crc32 = crc32c(p, len); 696bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe} 697bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe 698d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboestatic void fill_crc64(struct verify_header *hdr, void *p, unsigned int len) 699d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboe{ 700546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_crc64 *vh = hdr_priv(hdr); 701546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe 702546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe vh->crc64 = crc64(p, len); 703d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboe} 704d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboe 705e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboestatic void fill_md5(struct verify_header *hdr, void *p, unsigned int len) 706e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe{ 707546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_md5 *vh = hdr_priv(hdr); 7088c432325c3df2075a77b27eab8a87704cf7b48eeJens Axboe struct md5_ctx md5_ctx = { 709546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe .hash = (uint32_t *) vh->md5_digest, 7108c432325c3df2075a77b27eab8a87704cf7b48eeJens Axboe }; 711e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 71261f821f16cf27d8742e09b3a137c933cd0085e9bJens Axboe md5_init(&md5_ctx); 713e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe md5_update(&md5_ctx, p, len); 714e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe} 715e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 716e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe/* 717e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe * fill body of io_u->buf with random data and add a header with the 718e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe * crc32 or md5 sum of that data. 719e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe */ 720e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboevoid populate_verify_io_u(struct thread_data *td, struct io_u *io_u) 721e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe{ 722baefa9beacca430b8e3275c07d2f46ef14654468Jens Axboe struct verify_header *hdr; 72395646108f83a34c0486d7f3ed19e99c1d93691e6Jens Axboe void *p = io_u->buf, *data; 7247437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis unsigned int hdr_inc, data_len, header_num = 0; 725e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 7269cc3d1506b9f7ce0a9dba38c32c418e54ea0b573Jens Axboe if (td->o.verify == VERIFY_NULL) 7279cc3d1506b9f7ce0a9dba38c32c418e54ea0b573Jens Axboe return; 7289cc3d1506b9f7ce0a9dba38c32c418e54ea0b573Jens Axboe 729cbe8d7561cf6d81d741d87eb7940db2a111d2144Radha Ramachandran fill_pattern(td, p, io_u->buflen, io_u); 730546a9142511875524850ac92776184fd9fb7196eShawn Lewis 731546a9142511875524850ac92776184fd9fb7196eShawn Lewis hdr_inc = io_u->buflen; 732a59e170d683f9ccbaa45648d7ae9a13b70e3822cJens Axboe if (td->o.verify_interval) 733a59e170d683f9ccbaa45648d7ae9a13b70e3822cJens Axboe hdr_inc = td->o.verify_interval; 734baefa9beacca430b8e3275c07d2f46ef14654468Jens Axboe 7355ec10eaad3b09875b91e19a20bbdfa06f2117562Jens Axboe for (; p < io_u->buf + io_u->buflen; p += hdr_inc) { 73695646108f83a34c0486d7f3ed19e99c1d93691e6Jens Axboe hdr = p; 7373f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis 7383f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis hdr->fio_magic = FIO_HDR_MAGIC; 7393f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis hdr->verify_type = td->o.verify; 740546a9142511875524850ac92776184fd9fb7196eShawn Lewis hdr->len = hdr_inc; 7418767783245faac99a7e1c330a1d6536e579c8402Jens Axboe data_len = hdr_inc - hdr_size(hdr); 7423f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis 7438767783245faac99a7e1c330a1d6536e579c8402Jens Axboe data = p + hdr_size(hdr); 7443f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis switch (td->o.verify) { 7453f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis case VERIFY_MD5: 746bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n", 747bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe io_u, hdr->len); 7483f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis fill_md5(hdr, data, data_len); 7493f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis break; 7503f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis case VERIFY_CRC64: 751bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n", 752bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe io_u, hdr->len); 7533f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis fill_crc64(hdr, data, data_len); 7543f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis break; 755bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe case VERIFY_CRC32C: 7563845591fadea480177223e28c9d1c03642d34f0eJens Axboe case VERIFY_CRC32C_INTEL: 757bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n", 758bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe io_u, hdr->len); 759bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe fill_crc32c(hdr, data, data_len); 760bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe break; 7613f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis case VERIFY_CRC32: 762bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n", 763bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe io_u, hdr->len); 7643f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis fill_crc32(hdr, data, data_len); 7653f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis break; 7663f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis case VERIFY_CRC16: 767bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n", 768bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe io_u, hdr->len); 7693f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis fill_crc16(hdr, data, data_len); 7703f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis break; 7713f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis case VERIFY_CRC7: 772bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n", 773bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe io_u, hdr->len); 7743f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis fill_crc7(hdr, data, data_len); 7753f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis break; 776cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe case VERIFY_SHA256: 777bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n", 778bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe io_u, hdr->len); 779cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe fill_sha256(hdr, data, data_len); 780cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe break; 781cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe case VERIFY_SHA512: 782bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n", 783bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe io_u, hdr->len); 784cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe fill_sha512(hdr, data, data_len); 785cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe break; 7867437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis case VERIFY_META: 787bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "fill meta io_u %p, len %u\n", 788bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe io_u, hdr->len); 7897437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis fill_meta(hdr, td, io_u, header_num); 7907437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis break; 7917c353ceb3257b132f4c98326046b42201e070731Jens Axboe case VERIFY_SHA1: 7927c353ceb3257b132f4c98326046b42201e070731Jens Axboe dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n", 7937c353ceb3257b132f4c98326046b42201e070731Jens Axboe io_u, hdr->len); 7947c353ceb3257b132f4c98326046b42201e070731Jens Axboe fill_sha1(hdr, data, data_len); 7957c353ceb3257b132f4c98326046b42201e070731Jens Axboe break; 7963f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis default: 7973f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis log_err("fio: bad verify type: %d\n", td->o.verify); 7983f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis assert(0); 7993f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis } 800a59e170d683f9ccbaa45648d7ae9a13b70e3822cJens Axboe if (td->o.verify_offset) 8018767783245faac99a7e1c330a1d6536e579c8402Jens Axboe memswp(p, p + td->o.verify_offset, hdr_size(hdr)); 8027437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis header_num++; 803e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe } 804e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe} 805e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 806e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboeint get_next_verify(struct thread_data *td, struct io_u *io_u) 807e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe{ 8088de8f047bd025f12d23cfc3fc1793434c6d8ff94Jens Axboe struct io_piece *ipo = NULL; 809e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 810d2d7fa53c18f1fc3fb89f3fe20df9b39986bde72Jens Axboe /* 811d2d7fa53c18f1fc3fb89f3fe20df9b39986bde72Jens Axboe * this io_u is from a requeue, we already filled the offsets 812d2d7fa53c18f1fc3fb89f3fe20df9b39986bde72Jens Axboe */ 813d2d7fa53c18f1fc3fb89f3fe20df9b39986bde72Jens Axboe if (io_u->file) 814d2d7fa53c18f1fc3fb89f3fe20df9b39986bde72Jens Axboe return 0; 815d2d7fa53c18f1fc3fb89f3fe20df9b39986bde72Jens Axboe 8168de8f047bd025f12d23cfc3fc1793434c6d8ff94Jens Axboe if (!RB_EMPTY_ROOT(&td->io_hist_tree)) { 8178de8f047bd025f12d23cfc3fc1793434c6d8ff94Jens Axboe struct rb_node *n = rb_first(&td->io_hist_tree); 818e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 8198de8f047bd025f12d23cfc3fc1793434c6d8ff94Jens Axboe ipo = rb_entry(n, struct io_piece, rb_node); 8204b87898e8d76aaf05baec83077a11311c1447397Jens Axboe rb_erase(n, &td->io_hist_tree); 821a917a8b3dfeefdd7007ba2f46f21fc145574309dJens Axboe assert(ipo->flags & IP_F_ONRB); 822a917a8b3dfeefdd7007ba2f46f21fc145574309dJens Axboe ipo->flags &= ~IP_F_ONRB; 82301743ee1718e7ec4b16ae3e53c8f64900c6052ccJens Axboe } else if (!flist_empty(&td->io_hist_list)) { 82401743ee1718e7ec4b16ae3e53c8f64900c6052ccJens Axboe ipo = flist_entry(td->io_hist_list.next, struct io_piece, list); 82501743ee1718e7ec4b16ae3e53c8f64900c6052ccJens Axboe flist_del(&ipo->list); 826a917a8b3dfeefdd7007ba2f46f21fc145574309dJens Axboe assert(ipo->flags & IP_F_ONLIST); 827a917a8b3dfeefdd7007ba2f46f21fc145574309dJens Axboe ipo->flags &= ~IP_F_ONLIST; 8288de8f047bd025f12d23cfc3fc1793434c6d8ff94Jens Axboe } 829e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 8308de8f047bd025f12d23cfc3fc1793434c6d8ff94Jens Axboe if (ipo) { 8310d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe td->io_hist_len--; 8320d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe 833e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe io_u->offset = ipo->offset; 834e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe io_u->buflen = ipo->len; 83536167d82e5f49dee91c6d2cd426068edee90e36fJens Axboe io_u->file = ipo->file; 83697af62cec418cd722fb43c6010d6430534e12353Jens Axboe 8370d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe if (ipo->flags & IP_F_TRIMMED) 8380d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe io_u->flags |= IO_U_F_TRIMMED; 8390d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe 840d6aed795f2e3e403828abf60874dd2d6e8342a1bJens Axboe if (!fio_file_open(io_u->file)) { 84197af62cec418cd722fb43c6010d6430534e12353Jens Axboe int r = td_io_open_file(td, io_u->file); 84297af62cec418cd722fb43c6010d6430534e12353Jens Axboe 843bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe if (r) { 844bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "failed file %s open\n", 845bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe io_u->file->file_name); 84697af62cec418cd722fb43c6010d6430534e12353Jens Axboe return 1; 847bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe } 84897af62cec418cd722fb43c6010d6430534e12353Jens Axboe } 84997af62cec418cd722fb43c6010d6430534e12353Jens Axboe 85097af62cec418cd722fb43c6010d6430534e12353Jens Axboe get_file(ipo->file); 851d6aed795f2e3e403828abf60874dd2d6e8342a1bJens Axboe assert(fio_file_open(io_u->file)); 852e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe io_u->ddir = DDIR_READ; 85336167d82e5f49dee91c6d2cd426068edee90e36fJens Axboe io_u->xfer_buf = io_u->buf; 85436167d82e5f49dee91c6d2cd426068edee90e36fJens Axboe io_u->xfer_buflen = io_u->buflen; 8550d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe 8560d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe remove_trim_entry(td, ipo); 857e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe free(ipo); 858bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u); 859e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe return 0; 860e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe } 861e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 862bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "get_next_verify: empty\n"); 863e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe return 1; 864e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe} 865e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 866e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboestatic void *verify_async_thread(void *data) 867e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe{ 868e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe struct thread_data *td = data; 869e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe struct io_u *io_u; 870e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe int ret = 0; 871e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 872e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe if (td->o.verify_cpumask_set && 873e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe fio_setaffinity(td->pid, td->o.verify_cpumask)) { 874e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe log_err("fio: failed setting verify thread affinity\n"); 875e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe goto done; 876e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe } 877e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 878e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe do { 879e53ab27c7425241d64e8d64ad77850a776c01375Jens Axboe FLIST_HEAD(list); 880e53ab27c7425241d64e8d64ad77850a776c01375Jens Axboe 881e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe read_barrier(); 882e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe if (td->verify_thread_exit) 883e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe break; 884e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 885e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_mutex_lock(&td->io_u_lock); 886e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 887e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe while (flist_empty(&td->verify_list) && 888e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe !td->verify_thread_exit) { 889b36e298b5bdfc31ffc9a4c14e94a9c5c748888b0Jens Axboe ret = pthread_cond_wait(&td->verify_cond, 890b36e298b5bdfc31ffc9a4c14e94a9c5c748888b0Jens Axboe &td->io_u_lock); 891e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe if (ret) { 892e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_mutex_unlock(&td->io_u_lock); 893e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe break; 894e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe } 895e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe } 896e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 897e53ab27c7425241d64e8d64ad77850a776c01375Jens Axboe flist_splice_init(&td->verify_list, &list); 898e53ab27c7425241d64e8d64ad77850a776c01375Jens Axboe pthread_mutex_unlock(&td->io_u_lock); 899e53ab27c7425241d64e8d64ad77850a776c01375Jens Axboe 900e53ab27c7425241d64e8d64ad77850a776c01375Jens Axboe if (flist_empty(&list)) 901e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe continue; 902e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 903e53ab27c7425241d64e8d64ad77850a776c01375Jens Axboe while (!flist_empty(&list)) { 904e53ab27c7425241d64e8d64ad77850a776c01375Jens Axboe io_u = flist_entry(list.next, struct io_u, list); 905e53ab27c7425241d64e8d64ad77850a776c01375Jens Axboe flist_del_init(&io_u->list); 906e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 907d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe ret = verify_io_u(td, io_u); 908e53ab27c7425241d64e8d64ad77850a776c01375Jens Axboe put_io_u(td, io_u); 909d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe if (!ret) 910d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe continue; 911d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe if (td->o.continue_on_error && 912d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe td_non_fatal_error(ret)) { 913d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe update_error_count(td, ret); 914d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe td_clear_error(td); 915d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe ret = 0; 916d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe } 917e53ab27c7425241d64e8d64ad77850a776c01375Jens Axboe } 918e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe } while (!ret); 919e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 920d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe if (ret) { 921d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe td_verror(td, ret, "async_verify"); 922f3e6cb95fecf867ec46e3dfd59ef1a1db1b6ad2aJens Axboe if (td->o.verify_fatal) 923f3e6cb95fecf867ec46e3dfd59ef1a1db1b6ad2aJens Axboe td->terminate = 1; 924d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe } 925d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe 926e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboedone: 927e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_mutex_lock(&td->io_u_lock); 928e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe td->nr_verify_threads--; 929e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_mutex_unlock(&td->io_u_lock); 930e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 931e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_cond_signal(&td->free_cond); 932e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe return NULL; 933e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe} 934e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 935e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboeint verify_async_init(struct thread_data *td) 936e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe{ 937e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe int i, ret; 938304a47c7d94f407cc72a87025679a67f02288447bart Van Assche pthread_attr_t attr; 939304a47c7d94f407cc72a87025679a67f02288447bart Van Assche 940304a47c7d94f407cc72a87025679a67f02288447bart Van Assche pthread_attr_init(&attr); 941304a47c7d94f407cc72a87025679a67f02288447bart Van Assche pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN); 942e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 943e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe td->verify_thread_exit = 0; 944e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 945e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async); 946e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe for (i = 0; i < td->o.verify_async; i++) { 947304a47c7d94f407cc72a87025679a67f02288447bart Van Assche ret = pthread_create(&td->verify_threads[i], &attr, 948e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe verify_async_thread, td); 949e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe if (ret) { 950e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe log_err("fio: async verify creation failed: %s\n", 951e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe strerror(ret)); 952e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe break; 953e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe } 954e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe ret = pthread_detach(td->verify_threads[i]); 955e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe if (ret) { 956e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe log_err("fio: async verify thread detach failed: %s\n", 957e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe strerror(ret)); 958e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe break; 959e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe } 960e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe td->nr_verify_threads++; 961e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe } 962e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 963304a47c7d94f407cc72a87025679a67f02288447bart Van Assche pthread_attr_destroy(&attr); 964304a47c7d94f407cc72a87025679a67f02288447bart Van Assche 965e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe if (i != td->o.verify_async) { 966e40823b10920f679d84f3f2222e319c168b1e7daJens Axboe log_err("fio: only %d verify threads started, exiting\n", i); 967e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe td->verify_thread_exit = 1; 968e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe write_barrier(); 969e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_cond_broadcast(&td->verify_cond); 970e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe return 1; 971e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe } 972e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 973e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe return 0; 974e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe} 975e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 976e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboevoid verify_async_exit(struct thread_data *td) 977e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe{ 978e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe td->verify_thread_exit = 1; 979e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe write_barrier(); 980e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_cond_broadcast(&td->verify_cond); 981e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 982e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_mutex_lock(&td->io_u_lock); 983e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 984e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe while (td->nr_verify_threads) 985e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_cond_wait(&td->free_cond, &td->io_u_lock); 986e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 987e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_mutex_unlock(&td->io_u_lock); 988e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe free(td->verify_threads); 989e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe td->verify_threads = NULL; 990e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe} 991