1e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe/* 2e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe * IO verification helpers 3e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe */ 4e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe#include <unistd.h> 5e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe#include <fcntl.h> 6e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe#include <string.h> 797af62cec418cd722fb43c6010d6430534e12353Jens Axboe#include <assert.h> 8e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe#include <pthread.h> 979402a12eecf371238f2326c9e8850eedd269b7fJens Axboe#include <libgen.h> 10e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 11e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe#include "fio.h" 124f5af7b2370a6d3e64bc5128905c1aa8b0dc51b0Jens Axboe#include "verify.h" 130d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe#include "trim.h" 14637ef8d9f7645135cf4829894d1e3983cd7a042eJens Axboe#include "lib/rand.h" 1551aa2da8cf422a06ddfa1ce673f3bfc03f96b86eJens Axboe#include "lib/hweight.h" 16e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 17eef6eea1b935a67f1ae26b38e06a69d4410a12aaJens Axboe#include "crc/md5.h" 18eef6eea1b935a67f1ae26b38e06a69d4410a12aaJens Axboe#include "crc/crc64.h" 19eef6eea1b935a67f1ae26b38e06a69d4410a12aaJens Axboe#include "crc/crc32.h" 20bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe#include "crc/crc32c.h" 21eef6eea1b935a67f1ae26b38e06a69d4410a12aaJens Axboe#include "crc/crc16.h" 22eef6eea1b935a67f1ae26b38e06a69d4410a12aaJens Axboe#include "crc/crc7.h" 23eef6eea1b935a67f1ae26b38e06a69d4410a12aaJens Axboe#include "crc/sha256.h" 24eef6eea1b935a67f1ae26b38e06a69d4410a12aaJens Axboe#include "crc/sha512.h" 257c353ceb3257b132f4c98326046b42201e070731Jens Axboe#include "crc/sha1.h" 26844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe#include "crc/xxhash.h" 27cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 287d9fb455aadc0c0363489591775496f27f4a560aJens Axboestatic void populate_hdr(struct thread_data *td, struct io_u *io_u, 297d9fb455aadc0c0363489591775496f27f4a560aJens Axboe struct verify_header *hdr, unsigned int header_num, 307d9fb455aadc0c0363489591775496f27f4a560aJens Axboe unsigned int header_len); 317d9fb455aadc0c0363489591775496f27f4a560aJens Axboe 32ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboestatic void fill_pattern(struct thread_data *td, void *p, unsigned int len, 33ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe char *pattern, unsigned int pattern_bytes) 3490059d65bde4804e4da07401ad6902c1d956b592Jens Axboe{ 35ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe switch (pattern_bytes) { 3690059d65bde4804e4da07401ad6902c1d956b592Jens Axboe case 0: 37ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe assert(0); 3890059d65bde4804e4da07401ad6902c1d956b592Jens Axboe break; 3990059d65bde4804e4da07401ad6902c1d956b592Jens Axboe case 1: 40bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "fill verify pattern b=0 len=%u\n", len); 41ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe memset(p, pattern[0], len); 4290059d65bde4804e4da07401ad6902c1d956b592Jens Axboe break; 430e92f873a3d542d2b237d68fd9391c92a85fa0b8Radha Ramachandran default: { 440e92f873a3d542d2b237d68fd9391c92a85fa0b8Radha Ramachandran unsigned int i = 0, size = 0; 4590059d65bde4804e4da07401ad6902c1d956b592Jens Axboe unsigned char *b = p; 4690059d65bde4804e4da07401ad6902c1d956b592Jens Axboe 47bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "fill verify pattern b=%d len=%u\n", 48ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe pattern_bytes, len); 49bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe 5090059d65bde4804e4da07401ad6902c1d956b592Jens Axboe while (i < len) { 51ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe size = pattern_bytes; 520e92f873a3d542d2b237d68fd9391c92a85fa0b8Radha Ramachandran if (size > (len - i)) 530e92f873a3d542d2b237d68fd9391c92a85fa0b8Radha Ramachandran size = len - i; 54ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe memcpy(b+i, pattern, size); 550e92f873a3d542d2b237d68fd9391c92a85fa0b8Radha Ramachandran i += size; 5690059d65bde4804e4da07401ad6902c1d956b592Jens Axboe } 5790059d65bde4804e4da07401ad6902c1d956b592Jens Axboe break; 5890059d65bde4804e4da07401ad6902c1d956b592Jens Axboe } 5990059d65bde4804e4da07401ad6902c1d956b592Jens Axboe } 6090059d65bde4804e4da07401ad6902c1d956b592Jens Axboe} 6190059d65bde4804e4da07401ad6902c1d956b592Jens Axboe 62ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboevoid fill_buffer_pattern(struct thread_data *td, void *p, unsigned int len) 63ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe{ 64ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe fill_pattern(td, p, len, td->o.buffer_pattern, td->o.buffer_pattern_bytes); 65ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe} 66ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe 67ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboevoid fill_verify_pattern(struct thread_data *td, void *p, unsigned int len, 68ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe struct io_u *io_u, unsigned long seed, int use_seed) 69ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe{ 70ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe if (!td->o.verify_pattern_bytes) { 71ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe dprint(FD_VERIFY, "fill random bytes len=%u\n", len); 72ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe 73ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe if (use_seed) 74ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe __fill_random_buf(p, len, seed); 75ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe else 76c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat io_u->rand_seed = fill_random_buf(&td->__verify_state, p, len); 77ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe return; 78ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe } 79c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat 80ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe if (io_u->buf_filled_len >= len) { 81ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe dprint(FD_VERIFY, "using already filled verify pattern b=%d len=%u\n", 82ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe td->o.verify_pattern_bytes, len); 83ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe return; 84ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe } 85ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe 86ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe fill_pattern(td, p, len, td->o.verify_pattern, td->o.verify_pattern_bytes); 87ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe 88ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe io_u->buf_filled_len = len; 89ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe} 90ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe 91cfbcd0dc2fcbf0f9bfe92ff8f4ab17f21e46d21eJens Axboestatic unsigned int get_hdr_inc(struct thread_data *td, struct io_u *io_u) 92cfbcd0dc2fcbf0f9bfe92ff8f4ab17f21e46d21eJens Axboe{ 93cfbcd0dc2fcbf0f9bfe92ff8f4ab17f21e46d21eJens Axboe unsigned int hdr_inc; 94cfbcd0dc2fcbf0f9bfe92ff8f4ab17f21e46d21eJens Axboe 95cfbcd0dc2fcbf0f9bfe92ff8f4ab17f21e46d21eJens Axboe hdr_inc = io_u->buflen; 968a99fdf63e1350fc2c35794e1c1f779885905b29Jens Axboe if (td->o.verify_interval && td->o.verify_interval <= io_u->buflen) 97cfbcd0dc2fcbf0f9bfe92ff8f4ab17f21e46d21eJens Axboe hdr_inc = td->o.verify_interval; 98cfbcd0dc2fcbf0f9bfe92ff8f4ab17f21e46d21eJens Axboe 99cfbcd0dc2fcbf0f9bfe92ff8f4ab17f21e46d21eJens Axboe return hdr_inc; 100cfbcd0dc2fcbf0f9bfe92ff8f4ab17f21e46d21eJens Axboe} 101cfbcd0dc2fcbf0f9bfe92ff8f4ab17f21e46d21eJens Axboe 102c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboestatic void fill_pattern_headers(struct thread_data *td, struct io_u *io_u, 103c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe unsigned long seed, int use_seed) 104c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe{ 105c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe unsigned int hdr_inc, header_num; 106c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe struct verify_header *hdr; 107c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe void *p = io_u->buf; 108c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe 109ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe fill_verify_pattern(td, p, io_u->buflen, io_u, seed, use_seed); 110c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe 111cfbcd0dc2fcbf0f9bfe92ff8f4ab17f21e46d21eJens Axboe hdr_inc = get_hdr_inc(td, io_u); 112c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe header_num = 0; 113c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe for (; p < io_u->buf + io_u->buflen; p += hdr_inc) { 114c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe hdr = p; 115c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe populate_hdr(td, io_u, hdr, header_num, hdr_inc); 116c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe header_num++; 117c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe } 118c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe} 119c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe 1204764aec9fe5988281bdc114c13ac771fb16713ccJens Axboestatic void memswp(void *buf1, void *buf2, unsigned int len) 121546a9142511875524850ac92776184fd9fb7196eShawn Lewis{ 122dee6de74152299f2a7df07ae971a53441b971dafShawn Lewis char swap[200]; 123dee6de74152299f2a7df07ae971a53441b971dafShawn Lewis 124dee6de74152299f2a7df07ae971a53441b971dafShawn Lewis assert(len <= sizeof(swap)); 12590059d65bde4804e4da07401ad6902c1d956b592Jens Axboe 126546a9142511875524850ac92776184fd9fb7196eShawn Lewis memcpy(&swap, buf1, len); 127546a9142511875524850ac92776184fd9fb7196eShawn Lewis memcpy(buf1, buf2, len); 128546a9142511875524850ac92776184fd9fb7196eShawn Lewis memcpy(buf2, &swap, len); 129546a9142511875524850ac92776184fd9fb7196eShawn Lewis} 130546a9142511875524850ac92776184fd9fb7196eShawn Lewis 131e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboestatic void hexdump(void *buffer, int len) 132e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe{ 133e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe unsigned char *p = buffer; 134e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe int i; 135e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 136e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe for (i = 0; i < len; i++) 137bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_err("%02x", p[i]); 138bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_err("\n"); 139e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe} 140e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 141d9f2caf3de3f2ba6f8ecd8a2688e1eb7715bc2aaJens Axboe/* 142de8f6de97438d5664cd8765e60102b9109a273e2Anatol Pomozov * Prepare for separation of verify_header and checksum header 1438767783245faac99a7e1c330a1d6536e579c8402Jens Axboe */ 144546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboestatic inline unsigned int __hdr_size(int verify_type) 1458767783245faac99a7e1c330a1d6536e579c8402Jens Axboe{ 14603e20d687566753b90383571e5e152c5142bdffdBruce Cran unsigned int len = 0; 1478767783245faac99a7e1c330a1d6536e579c8402Jens Axboe 148546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe switch (verify_type) { 149546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe case VERIFY_NONE: 150546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe case VERIFY_NULL: 151546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe len = 0; 152546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe break; 153546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe case VERIFY_MD5: 154546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe len = sizeof(struct vhdr_md5); 155546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe break; 156546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe case VERIFY_CRC64: 157546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe len = sizeof(struct vhdr_crc64); 158546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe break; 159bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe case VERIFY_CRC32C: 160546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe case VERIFY_CRC32: 1613845591fadea480177223e28c9d1c03642d34f0eJens Axboe case VERIFY_CRC32C_INTEL: 162546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe len = sizeof(struct vhdr_crc32); 163546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe break; 164546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe case VERIFY_CRC16: 165546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe len = sizeof(struct vhdr_crc16); 166546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe break; 167546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe case VERIFY_CRC7: 168546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe len = sizeof(struct vhdr_crc7); 169546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe break; 170546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe case VERIFY_SHA256: 171546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe len = sizeof(struct vhdr_sha256); 172546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe break; 173546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe case VERIFY_SHA512: 174546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe len = sizeof(struct vhdr_sha512); 175546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe break; 176844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe case VERIFY_XXHASH: 177844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe len = sizeof(struct vhdr_xxhash); 178844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe break; 1797437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis case VERIFY_META: 1807437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis len = sizeof(struct vhdr_meta); 1817437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis break; 1827c353ceb3257b132f4c98326046b42201e070731Jens Axboe case VERIFY_SHA1: 1837c353ceb3257b132f4c98326046b42201e070731Jens Axboe len = sizeof(struct vhdr_sha1); 1847c353ceb3257b132f4c98326046b42201e070731Jens Axboe break; 18592bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe case VERIFY_PATTERN: 18692bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe len = 0; 18792bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe break; 188546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe default: 189546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe log_err("fio: unknown verify header!\n"); 190546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe assert(0); 191546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe } 192546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe 193546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe return len + sizeof(struct verify_header); 1948767783245faac99a7e1c330a1d6536e579c8402Jens Axboe} 1958767783245faac99a7e1c330a1d6536e579c8402Jens Axboe 1968767783245faac99a7e1c330a1d6536e579c8402Jens Axboestatic inline unsigned int hdr_size(struct verify_header *hdr) 1978767783245faac99a7e1c330a1d6536e579c8402Jens Axboe{ 198546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe return __hdr_size(hdr->verify_type); 199546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe} 200546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe 201546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboestatic void *hdr_priv(struct verify_header *hdr) 202546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe{ 203546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe void *priv = hdr; 204546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe 205546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe return priv + sizeof(struct verify_header); 2068767783245faac99a7e1c330a1d6536e579c8402Jens Axboe} 2078767783245faac99a7e1c330a1d6536e579c8402Jens Axboe 2088767783245faac99a7e1c330a1d6536e579c8402Jens Axboe/* 209936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe * Verify container, pass info to verify handlers and allow them to 210936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe * pass info back in case of error 211936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe */ 212936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestruct vcont { 213936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe /* 214936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe * Input 215936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe */ 216936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe struct io_u *io_u; 217936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe unsigned int hdr_num; 2187d9fb455aadc0c0363489591775496f27f4a560aJens Axboe struct thread_data *td; 219936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe 220936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe /* 221936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe * Output, only valid in case of error 222936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe */ 223bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe const char *name; 224bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe void *good_crc; 225bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe void *bad_crc; 226936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe unsigned int crc_len; 227936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe}; 228936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe 229dacbbb8875c09e982d59e1c1a40879af81842a42Jens Axboe#define DUMP_BUF_SZ 255 230dacbbb8875c09e982d59e1c1a40879af81842a42Jens Axboestatic int dump_buf_warned; 231dacbbb8875c09e982d59e1c1a40879af81842a42Jens Axboe 232c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboestatic void dump_buf(char *buf, unsigned int len, unsigned long long offset, 233c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe const char *type, struct fio_file *f) 2347d9fb455aadc0c0363489591775496f27f4a560aJens Axboe{ 235dacbbb8875c09e982d59e1c1a40879af81842a42Jens Axboe char *ptr, fname[DUMP_BUF_SZ]; 236dacbbb8875c09e982d59e1c1a40879af81842a42Jens Axboe size_t buf_left = DUMP_BUF_SZ; 2377d9fb455aadc0c0363489591775496f27f4a560aJens Axboe int ret, fd; 2387d9fb455aadc0c0363489591775496f27f4a560aJens Axboe 2396f260b317f4cec3027f79b8b329dd3f98c3906acJens Axboe ptr = strdup(f->file_name); 240c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe 241dacbbb8875c09e982d59e1c1a40879af81842a42Jens Axboe fname[DUMP_BUF_SZ - 1] = '\0'; 242dacbbb8875c09e982d59e1c1a40879af81842a42Jens Axboe strncpy(fname, basename(ptr), DUMP_BUF_SZ - 1); 243dacbbb8875c09e982d59e1c1a40879af81842a42Jens Axboe 244dacbbb8875c09e982d59e1c1a40879af81842a42Jens Axboe buf_left -= strlen(fname); 245dacbbb8875c09e982d59e1c1a40879af81842a42Jens Axboe if (buf_left <= 0) { 246dacbbb8875c09e982d59e1c1a40879af81842a42Jens Axboe if (!dump_buf_warned) { 247dacbbb8875c09e982d59e1c1a40879af81842a42Jens Axboe log_err("fio: verify failure dump buffer too small\n"); 248dacbbb8875c09e982d59e1c1a40879af81842a42Jens Axboe dump_buf_warned = 1; 249dacbbb8875c09e982d59e1c1a40879af81842a42Jens Axboe } 250dacbbb8875c09e982d59e1c1a40879af81842a42Jens Axboe free(ptr); 251dacbbb8875c09e982d59e1c1a40879af81842a42Jens Axboe return; 252dacbbb8875c09e982d59e1c1a40879af81842a42Jens Axboe } 253dacbbb8875c09e982d59e1c1a40879af81842a42Jens Axboe 254dacbbb8875c09e982d59e1c1a40879af81842a42Jens Axboe snprintf(fname + strlen(fname), buf_left, ".%llu.%s", offset, type); 2557d9fb455aadc0c0363489591775496f27f4a560aJens Axboe 2567d9fb455aadc0c0363489591775496f27f4a560aJens Axboe fd = open(fname, O_CREAT | O_TRUNC | O_WRONLY, 0644); 2577d9fb455aadc0c0363489591775496f27f4a560aJens Axboe if (fd < 0) { 2587d9fb455aadc0c0363489591775496f27f4a560aJens Axboe perror("open verify buf file"); 2597d9fb455aadc0c0363489591775496f27f4a560aJens Axboe return; 2607d9fb455aadc0c0363489591775496f27f4a560aJens Axboe } 2617d9fb455aadc0c0363489591775496f27f4a560aJens Axboe 2627d9fb455aadc0c0363489591775496f27f4a560aJens Axboe while (len) { 2637d9fb455aadc0c0363489591775496f27f4a560aJens Axboe ret = write(fd, buf, len); 2647d9fb455aadc0c0363489591775496f27f4a560aJens Axboe if (!ret) 2657d9fb455aadc0c0363489591775496f27f4a560aJens Axboe break; 2667d9fb455aadc0c0363489591775496f27f4a560aJens Axboe else if (ret < 0) { 2677d9fb455aadc0c0363489591775496f27f4a560aJens Axboe perror("write verify buf file"); 2687d9fb455aadc0c0363489591775496f27f4a560aJens Axboe break; 2697d9fb455aadc0c0363489591775496f27f4a560aJens Axboe } 2707d9fb455aadc0c0363489591775496f27f4a560aJens Axboe len -= ret; 2717d9fb455aadc0c0363489591775496f27f4a560aJens Axboe buf += ret; 2727d9fb455aadc0c0363489591775496f27f4a560aJens Axboe } 2737d9fb455aadc0c0363489591775496f27f4a560aJens Axboe 2747d9fb455aadc0c0363489591775496f27f4a560aJens Axboe close(fd); 275c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe log_err(" %s data dumped as %s\n", type, fname); 2766f260b317f4cec3027f79b8b329dd3f98c3906acJens Axboe free(ptr); 2777d9fb455aadc0c0363489591775496f27f4a560aJens Axboe} 2787d9fb455aadc0c0363489591775496f27f4a560aJens Axboe 279c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe/* 280c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe * Dump the contents of the read block and re-generate the correct data 281c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe * and dump that too. 282c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe */ 2837d9fb455aadc0c0363489591775496f27f4a560aJens Axboestatic void dump_verify_buffers(struct verify_header *hdr, struct vcont *vc) 2847d9fb455aadc0c0363489591775496f27f4a560aJens Axboe{ 2857d9fb455aadc0c0363489591775496f27f4a560aJens Axboe struct thread_data *td = vc->td; 2867d9fb455aadc0c0363489591775496f27f4a560aJens Axboe struct io_u *io_u = vc->io_u; 2877d9fb455aadc0c0363489591775496f27f4a560aJens Axboe unsigned long hdr_offset; 2887d9fb455aadc0c0363489591775496f27f4a560aJens Axboe struct io_u dummy; 289c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe void *buf; 2907d9fb455aadc0c0363489591775496f27f4a560aJens Axboe 2910dce9bc9b21a8d91aa55a29258dfaeffc57bd466Steven Lang if (!td->o.verify_dump) 2920dce9bc9b21a8d91aa55a29258dfaeffc57bd466Steven Lang return; 2930dce9bc9b21a8d91aa55a29258dfaeffc57bd466Steven Lang 294c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe /* 295c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe * Dump the contents we just read off disk 296c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe */ 2977d9fb455aadc0c0363489591775496f27f4a560aJens Axboe hdr_offset = vc->hdr_num * hdr->len; 2987d9fb455aadc0c0363489591775496f27f4a560aJens Axboe 299c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe dump_buf(io_u->buf + hdr_offset, hdr->len, io_u->offset + hdr_offset, 300c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe "received", vc->io_u->file); 3017d9fb455aadc0c0363489591775496f27f4a560aJens Axboe 302c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe /* 303c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe * Allocate a new buf and re-generate the original data 304c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe */ 305c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe buf = malloc(io_u->buflen); 3067d9fb455aadc0c0363489591775496f27f4a560aJens Axboe dummy = *io_u; 307c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe dummy.buf = buf; 3084aae38a645f3315ca2f8cc57a7ec4110ba7d8319Jens Axboe dummy.rand_seed = hdr->rand_seed; 309cfbcd0dc2fcbf0f9bfe92ff8f4ab17f21e46d21eJens Axboe dummy.buf_filled_len = 0; 3100d81daf9d164c4bf829f4911e1f7b211c2e661fbJosef Bacik dummy.buflen = io_u->buflen; 3117d9fb455aadc0c0363489591775496f27f4a560aJens Axboe 312c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe fill_pattern_headers(td, &dummy, hdr->rand_seed, 1); 3137d9fb455aadc0c0363489591775496f27f4a560aJens Axboe 314c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe dump_buf(buf + hdr_offset, hdr->len, io_u->offset + hdr_offset, 315c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe "expected", vc->io_u->file); 3167d9fb455aadc0c0363489591775496f27f4a560aJens Axboe free(buf); 3177d9fb455aadc0c0363489591775496f27f4a560aJens Axboe} 3187d9fb455aadc0c0363489591775496f27f4a560aJens Axboe 319bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboestatic void log_verify_failure(struct verify_header *hdr, struct vcont *vc) 320bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe{ 321bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe unsigned long long offset; 322bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe 323bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe offset = vc->io_u->offset; 324bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe offset += vc->hdr_num * hdr->len; 32532c17adf7f75da1f0957d4691633fea60259910fJens Axboe log_err("%.8s: verify failed at file %s offset %llu, length %u\n", 32632c17adf7f75da1f0957d4691633fea60259910fJens Axboe vc->name, vc->io_u->file->file_name, offset, hdr->len); 327bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe 328bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (vc->good_crc && vc->bad_crc) { 329bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_err(" Expected CRC: "); 330bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe hexdump(vc->good_crc, vc->crc_len); 331bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_err(" Received CRC: "); 332bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe hexdump(vc->bad_crc, vc->crc_len); 333bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe } 3347d9fb455aadc0c0363489591775496f27f4a560aJens Axboe 3357d9fb455aadc0c0363489591775496f27f4a560aJens Axboe dump_verify_buffers(hdr, vc); 336bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe} 337bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe 338936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe/* 339d9f2caf3de3f2ba6f8ecd8a2688e1eb7715bc2aaJens Axboe * Return data area 'header_num' 340d9f2caf3de3f2ba6f8ecd8a2688e1eb7715bc2aaJens Axboe */ 341936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestatic inline void *io_u_verify_off(struct verify_header *hdr, struct vcont *vc) 342d9f2caf3de3f2ba6f8ecd8a2688e1eb7715bc2aaJens Axboe{ 343936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe return vc->io_u->buf + vc->hdr_num * hdr->len + hdr_size(hdr); 344d9f2caf3de3f2ba6f8ecd8a2688e1eb7715bc2aaJens Axboe} 345d9f2caf3de3f2ba6f8ecd8a2688e1eb7715bc2aaJens Axboe 34692bf48d530ab09bae4058df711c4723c73da51b9Jens Axboestatic int verify_io_u_pattern(struct verify_header *hdr, struct vcont *vc) 34792bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe{ 34892bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe struct thread_data *td = vc->td; 34992bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe struct io_u *io_u = vc->io_u; 35092bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe char *buf, *pattern; 3512b13e716c0921356c0930522718e00b8df34293aJens Axboe unsigned int header_size = __hdr_size(td->o.verify); 3529a2a86d0f6f1425a6da666459ccd379e356a30d0Steven Lang unsigned int len, mod, i, size, pattern_size; 35392bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe 35492bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe pattern = td->o.verify_pattern; 3559a2a86d0f6f1425a6da666459ccd379e356a30d0Steven Lang pattern_size = td->o.verify_pattern_bytes; 3569a2a86d0f6f1425a6da666459ccd379e356a30d0Steven Lang if (pattern_size <= 1) 3579a2a86d0f6f1425a6da666459ccd379e356a30d0Steven Lang pattern_size = MAX_PATTERN_SIZE; 3582b13e716c0921356c0930522718e00b8df34293aJens Axboe buf = (void *) hdr + header_size; 3592b13e716c0921356c0930522718e00b8df34293aJens Axboe len = get_hdr_inc(td, io_u) - header_size; 3609a2a86d0f6f1425a6da666459ccd379e356a30d0Steven Lang mod = header_size % pattern_size; 3619a2a86d0f6f1425a6da666459ccd379e356a30d0Steven Lang 3629a2a86d0f6f1425a6da666459ccd379e356a30d0Steven Lang for (i = 0; i < len; i += size) { 3639a2a86d0f6f1425a6da666459ccd379e356a30d0Steven Lang size = pattern_size - mod; 3649a2a86d0f6f1425a6da666459ccd379e356a30d0Steven Lang if (size > (len - i)) 3659a2a86d0f6f1425a6da666459ccd379e356a30d0Steven Lang size = len - i; 3669a2a86d0f6f1425a6da666459ccd379e356a30d0Steven Lang if (memcmp(buf + i, pattern + mod, size)) 367e4ad68b1364aebfe949e2d6a649c7b1001ebca07Jens Axboe /* Let the slow compare find the first mismatch byte. */ 3689a2a86d0f6f1425a6da666459ccd379e356a30d0Steven Lang break; 3699a2a86d0f6f1425a6da666459ccd379e356a30d0Steven Lang mod = 0; 3709a2a86d0f6f1425a6da666459ccd379e356a30d0Steven Lang } 37192bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe 3729a2a86d0f6f1425a6da666459ccd379e356a30d0Steven Lang for (; i < len; i++) { 37392bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe if (buf[i] != pattern[mod]) { 37492bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe unsigned int bits; 37592bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe 37692bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe bits = hweight8(buf[i] ^ pattern[mod]); 37792bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe log_err("fio: got pattern %x, wanted %x. Bad bits %d\n", 37892bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe buf[i], pattern[mod], bits); 37992bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe log_err("fio: bad pattern block offset %u\n", i); 38092bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe dump_verify_buffers(hdr, vc); 38192bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe return EILSEQ; 38292bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe } 38392bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe mod++; 38492bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe if (mod == td->o.verify_pattern_bytes) 38592bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe mod = 0; 38692bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe } 38792bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe 38892bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe return 0; 38992bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe} 39092bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe 39192bf48d530ab09bae4058df711c4723c73da51b9Jens Axboestatic int verify_io_u_meta(struct verify_header *hdr, struct vcont *vc) 3927437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis{ 39392bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe struct thread_data *td = vc->td; 3947437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis struct vhdr_meta *vh = hdr_priv(hdr); 395936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe struct io_u *io_u = vc->io_u; 39692bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe int ret = EILSEQ; 3977437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis 398bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len); 399bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe 400bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (vh->offset == io_u->offset + vc->hdr_num * td->o.verify_interval) 40192bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe ret = 0; 40292bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe 40392bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe if (td->o.verify_pattern_bytes) 40492bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe ret |= verify_io_u_pattern(hdr, vc); 40592bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe 406da0a7bd224bb9331f27bb4b20394dd5c8fa3acb0Juan Casse /* 407da0a7bd224bb9331f27bb4b20394dd5c8fa3acb0Juan Casse * For read-only workloads, the program cannot be certain of the 408da0a7bd224bb9331f27bb4b20394dd5c8fa3acb0Juan Casse * last numberio written to a block. Checking of numberio will be done 409da0a7bd224bb9331f27bb4b20394dd5c8fa3acb0Juan Casse * only for workloads that write data. 410621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse * For verify_only, numberio will be checked in the last iteration when 411621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse * the correct state of numberio, that would have been written to each 412621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse * block in a previous run of fio, has been reached. 413da0a7bd224bb9331f27bb4b20394dd5c8fa3acb0Juan Casse */ 414da0a7bd224bb9331f27bb4b20394dd5c8fa3acb0Juan Casse if (td_write(td) || td_rw(td)) 415621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse if (!td->o.verify_only || td->o.loops == 0) 416621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse if (vh->numberio != io_u->numberio) 417621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse ret = EILSEQ; 418da0a7bd224bb9331f27bb4b20394dd5c8fa3acb0Juan Casse 41992bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe if (!ret) 420bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return 0; 4217437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis 422bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->name = "meta"; 423bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_verify_failure(hdr, vc); 42492bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe return ret; 4257437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis} 4267437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis 427844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboestatic int verify_io_u_xxhash(struct verify_header *hdr, struct vcont *vc) 428844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe{ 429844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe void *p = io_u_verify_off(hdr, vc); 430844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe struct vhdr_xxhash *vh = hdr_priv(hdr); 431844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe uint32_t hash; 432844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe void *state; 433844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe 434844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe dprint(FD_VERIFY, "xxhash verify io_u %p, len %u\n", vc->io_u, hdr->len); 435844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe 436844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe state = XXH32_init(1); 437844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe XXH32_update(state, p, hdr->len - hdr_size(hdr)); 438844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe hash = XXH32_digest(state); 439844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe 440844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe if (vh->hash == hash) 441844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe return 0; 442844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe 443844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe vc->name = "xxhash"; 444844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe vc->good_crc = &vh->hash; 445844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe vc->bad_crc = &hash; 446844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe vc->crc_len = sizeof(hash); 447844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe log_verify_failure(hdr, vc); 448844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe return EILSEQ; 449844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe} 450844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe 451936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestatic int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc) 452cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe{ 453936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe void *p = io_u_verify_off(hdr, vc); 454546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_sha512 *vh = hdr_priv(hdr); 455cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe uint8_t sha512[128]; 45625dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe struct fio_sha512_ctx sha512_ctx = { 457cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe .buf = sha512, 458cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe }; 459cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 460936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", vc->io_u, hdr->len); 461bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe 46225dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe fio_sha512_init(&sha512_ctx); 46325dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe fio_sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr)); 464cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 465bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (!memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512))) 466bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return 0; 467cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 468bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->name = "sha512"; 469bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->good_crc = vh->sha512; 470bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->bad_crc = sha512_ctx.buf; 471bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->crc_len = sizeof(vh->sha512); 472bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_verify_failure(hdr, vc); 473bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return EILSEQ; 474cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe} 475cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 476936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestatic int verify_io_u_sha256(struct verify_header *hdr, struct vcont *vc) 477cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe{ 478936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe void *p = io_u_verify_off(hdr, vc); 479546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_sha256 *vh = hdr_priv(hdr); 480bc77f56f79f6ae155e9eb26263ed686ef28b47ecJens Axboe uint8_t sha256[64]; 48125dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe struct fio_sha256_ctx sha256_ctx = { 482cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe .buf = sha256, 483cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe }; 484cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 485936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", vc->io_u, hdr->len); 486bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe 48725dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe fio_sha256_init(&sha256_ctx); 48825dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe fio_sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr)); 489cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 490bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (!memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256))) 491bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return 0; 492cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 493bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->name = "sha256"; 494bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->good_crc = vh->sha256; 495bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->bad_crc = sha256_ctx.buf; 496bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->crc_len = sizeof(vh->sha256); 497bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_verify_failure(hdr, vc); 498bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return EILSEQ; 499cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe} 500cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 501936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestatic int verify_io_u_sha1(struct verify_header *hdr, struct vcont *vc) 5027c353ceb3257b132f4c98326046b42201e070731Jens Axboe{ 503936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe void *p = io_u_verify_off(hdr, vc); 5047c353ceb3257b132f4c98326046b42201e070731Jens Axboe struct vhdr_sha1 *vh = hdr_priv(hdr); 5057c353ceb3257b132f4c98326046b42201e070731Jens Axboe uint32_t sha1[5]; 50625dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe struct fio_sha1_ctx sha1_ctx = { 5077c353ceb3257b132f4c98326046b42201e070731Jens Axboe .H = sha1, 5087c353ceb3257b132f4c98326046b42201e070731Jens Axboe }; 5097c353ceb3257b132f4c98326046b42201e070731Jens Axboe 510936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", vc->io_u, hdr->len); 5117c353ceb3257b132f4c98326046b42201e070731Jens Axboe 51225dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe fio_sha1_init(&sha1_ctx); 51325dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe fio_sha1_update(&sha1_ctx, p, hdr->len - hdr_size(hdr)); 5147c353ceb3257b132f4c98326046b42201e070731Jens Axboe 515bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (!memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1))) 516bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return 0; 5177c353ceb3257b132f4c98326046b42201e070731Jens Axboe 518bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->name = "sha1"; 519bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->good_crc = vh->sha1; 520bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->bad_crc = sha1_ctx.H; 521bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->crc_len = sizeof(vh->sha1); 522bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_verify_failure(hdr, vc); 523bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return EILSEQ; 5247c353ceb3257b132f4c98326046b42201e070731Jens Axboe} 5257c353ceb3257b132f4c98326046b42201e070731Jens Axboe 526936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestatic int verify_io_u_crc7(struct verify_header *hdr, struct vcont *vc) 5271e154bdbf14a1d352117aea057035235b66f0381Jens Axboe{ 528936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe void *p = io_u_verify_off(hdr, vc); 529546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_crc7 *vh = hdr_priv(hdr); 5301e154bdbf14a1d352117aea057035235b66f0381Jens Axboe unsigned char c; 5311e154bdbf14a1d352117aea057035235b66f0381Jens Axboe 532936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", vc->io_u, hdr->len); 533bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe 53425dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe c = fio_crc7(p, hdr->len - hdr_size(hdr)); 5351e154bdbf14a1d352117aea057035235b66f0381Jens Axboe 536bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (c == vh->crc7) 537bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return 0; 5381e154bdbf14a1d352117aea057035235b66f0381Jens Axboe 539bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->name = "crc7"; 540bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->good_crc = &vh->crc7; 541bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->bad_crc = &c; 542bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->crc_len = 1; 543bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_verify_failure(hdr, vc); 544bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return EILSEQ; 5451e154bdbf14a1d352117aea057035235b66f0381Jens Axboe} 5461e154bdbf14a1d352117aea057035235b66f0381Jens Axboe 547936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestatic int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc) 548969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboe{ 549936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe void *p = io_u_verify_off(hdr, vc); 550546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_crc16 *vh = hdr_priv(hdr); 551969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboe unsigned short c; 552969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboe 553936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len); 554bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe 55525dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe c = fio_crc16(p, hdr->len - hdr_size(hdr)); 556969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboe 557bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (c == vh->crc16) 558bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return 0; 559969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboe 560bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->name = "crc16"; 561bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->good_crc = &vh->crc16; 562bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->bad_crc = &c; 563bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->crc_len = 2; 564bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_verify_failure(hdr, vc); 565bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return EILSEQ; 566969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboe} 567969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboe 568936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestatic int verify_io_u_crc64(struct verify_header *hdr, struct vcont *vc) 569d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboe{ 570936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe void *p = io_u_verify_off(hdr, vc); 571546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_crc64 *vh = hdr_priv(hdr); 572d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboe unsigned long long c; 573d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboe 574936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", vc->io_u, hdr->len); 575bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe 57625dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe c = fio_crc64(p, hdr->len - hdr_size(hdr)); 577d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboe 578bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (c == vh->crc64) 579bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return 0; 580d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboe 581bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->name = "crc64"; 582bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->good_crc = &vh->crc64; 583bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->bad_crc = &c; 584bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->crc_len = 8; 585bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_verify_failure(hdr, vc); 586bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return EILSEQ; 587d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboe} 588d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboe 589936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestatic int verify_io_u_crc32(struct verify_header *hdr, struct vcont *vc) 590e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe{ 591936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe void *p = io_u_verify_off(hdr, vc); 592546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_crc32 *vh = hdr_priv(hdr); 593546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe uint32_t c; 594e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 595936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", vc->io_u, hdr->len); 596bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe 59725dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe c = fio_crc32(p, hdr->len - hdr_size(hdr)); 598e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 599bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (c == vh->crc32) 600bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return 0; 601e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 602bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->name = "crc32"; 603bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->good_crc = &vh->crc32; 604bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->bad_crc = &c; 605bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->crc_len = 4; 606bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_verify_failure(hdr, vc); 607bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return EILSEQ; 608e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe} 609e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 610936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestatic int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc) 611bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe{ 612936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe void *p = io_u_verify_off(hdr, vc); 613bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe struct vhdr_crc32 *vh = hdr_priv(hdr); 614bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe uint32_t c; 615bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe 616936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len); 617bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe 61825dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe c = fio_crc32c(p, hdr->len - hdr_size(hdr)); 619bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe 620bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (c == vh->crc32) 621bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return 0; 622bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe 623bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->name = "crc32c"; 624bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->good_crc = &vh->crc32; 625bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->bad_crc = &c; 626bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->crc_len = 4; 627bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_verify_failure(hdr, vc); 628bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return EILSEQ; 629bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe} 630bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe 631936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboestatic int verify_io_u_md5(struct verify_header *hdr, struct vcont *vc) 632e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe{ 633936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe void *p = io_u_verify_off(hdr, vc); 634546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_md5 *vh = hdr_priv(hdr); 6358c432325c3df2075a77b27eab8a87704cf7b48eeJens Axboe uint32_t hash[MD5_HASH_WORDS]; 63625dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe struct fio_md5_ctx md5_ctx = { 6378c432325c3df2075a77b27eab8a87704cf7b48eeJens Axboe .hash = hash, 6388c432325c3df2075a77b27eab8a87704cf7b48eeJens Axboe }; 639e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 640936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", vc->io_u, hdr->len); 641bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe 64225dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe fio_md5_init(&md5_ctx); 64325dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe fio_md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr)); 644e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 645bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe if (!memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash))) 646bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return 0; 647e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 648bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->name = "md5"; 649bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->good_crc = vh->md5_digest; 650bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->bad_crc = md5_ctx.hash; 651bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe vc->crc_len = sizeof(hash); 652bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe log_verify_failure(hdr, vc); 653bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe return EILSEQ; 654e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe} 655e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 656e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe/* 657e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe * Push IO verification to a separate thread 658e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe */ 659e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboeint verify_io_u_async(struct thread_data *td, struct io_u *io_u) 660e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe{ 661e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe if (io_u->file) 662e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe put_file_log(td, io_u->file); 663e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 664e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_mutex_lock(&td->io_u_lock); 665d7ee2a7d431c2cabdf56d04ca14805cad72cf985Steven Lang 6660c41214ff4e6f31f8df64aac37be7853aada6f1fRadha Ramachandran if (io_u->flags & IO_U_F_IN_CUR_DEPTH) { 6670c41214ff4e6f31f8df64aac37be7853aada6f1fRadha Ramachandran td->cur_depth--; 6680c41214ff4e6f31f8df64aac37be7853aada6f1fRadha Ramachandran io_u->flags &= ~IO_U_F_IN_CUR_DEPTH; 6690c41214ff4e6f31f8df64aac37be7853aada6f1fRadha Ramachandran } 6702ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe flist_add_tail(&io_u->verify_list, &td->verify_list); 6712ecc1b57721e3cb72bbf558bc169c97037fe3d0aJens Axboe io_u->flags |= IO_U_F_FREE_DEF; 672e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_mutex_unlock(&td->io_u_lock); 673e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 674e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_cond_signal(&td->verify_cond); 675e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe return 0; 676e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe} 677e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 6780d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboestatic int verify_trimmed_io_u(struct thread_data *td, struct io_u *io_u) 6790d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe{ 6800d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe static char zero_buf[1024]; 6810d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe unsigned int this_len, len; 6820d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe int ret = 0; 6830d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe void *p; 6840d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe 6850d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe if (!td->o.trim_zero) 6860d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe return 0; 6870d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe 6880d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe len = io_u->buflen; 6890d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe p = io_u->buf; 6900d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe do { 6910d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe this_len = sizeof(zero_buf); 6920d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe if (this_len > len) 6930d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe this_len = len; 6940d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe if (memcmp(p, zero_buf, this_len)) { 6950d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe ret = EILSEQ; 6960d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe break; 6970d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe } 6980d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe len -= this_len; 6990d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe p += this_len; 7000d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe } while (len); 7010d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe 7020d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe if (!ret) 7030d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe return 0; 7040d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe 705a917a8b3dfeefdd7007ba2f46f21fc145574309dJens Axboe log_err("trim: verify failed at file %s offset %llu, length %lu" 706a917a8b3dfeefdd7007ba2f46f21fc145574309dJens Axboe ", block offset %lu\n", 707a917a8b3dfeefdd7007ba2f46f21fc145574309dJens Axboe io_u->file->file_name, io_u->offset, io_u->buflen, 7082f68124f26e54233db41b462a257dabc48e5c32bJens Axboe (unsigned long) (p - io_u->buf)); 7090d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe return ret; 7100d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe} 7110d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe 7120ae2c6e1d4727515e35e682eae6776fe55fa0275Jens Axboestatic int verify_header(struct io_u *io_u, struct verify_header *hdr) 713f65d1c2663ac6007eac1a2063fe25b4275d013e4Jens Axboe{ 714f65d1c2663ac6007eac1a2063fe25b4275d013e4Jens Axboe void *p = hdr; 715f65d1c2663ac6007eac1a2063fe25b4275d013e4Jens Axboe uint32_t crc; 716f65d1c2663ac6007eac1a2063fe25b4275d013e4Jens Axboe 717ae38c0dcd570265aef09aa132ce27278d8909a1dJens Axboe if (hdr->magic != FIO_HDR_MAGIC) 718e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse return 1; 719e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse if (hdr->len > io_u->buflen) 720e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse return 2; 721e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse if (hdr->rand_seed != io_u->rand_seed) 722e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse return 3; 723ae38c0dcd570265aef09aa132ce27278d8909a1dJens Axboe 72425dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe crc = fio_crc32c(p, offsetof(struct verify_header, crc32)); 725f65d1c2663ac6007eac1a2063fe25b4275d013e4Jens Axboe if (crc == hdr->crc32) 726e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse return 0; 727f65d1c2663ac6007eac1a2063fe25b4275d013e4Jens Axboe log_err("fio: verify header crc %x, calculated %x\n", hdr->crc32, crc); 728e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse return 4; 729f65d1c2663ac6007eac1a2063fe25b4275d013e4Jens Axboe} 730f65d1c2663ac6007eac1a2063fe25b4275d013e4Jens Axboe 73136690c9b5b50736cec1fcd53100f01fd1181fdf4Jens Axboeint verify_io_u(struct thread_data *td, struct io_u *io_u) 732e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe{ 7333f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis struct verify_header *hdr; 7342b13e716c0921356c0930522718e00b8df34293aJens Axboe unsigned int header_size, hdr_inc, hdr_num = 0; 73595646108f83a34c0486d7f3ed19e99c1d93691e6Jens Axboe void *p; 736e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe int ret; 737e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 7381dcc049824b840df1ee0ef5d019fffa4037e1bb0Shawn Lewis if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ) 73936690c9b5b50736cec1fcd53100f01fd1181fdf4Jens Axboe return 0; 7400d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe if (io_u->flags & IO_U_F_TRIMMED) { 7410d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe ret = verify_trimmed_io_u(td, io_u); 7420d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe goto done; 7430d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe } 74436690c9b5b50736cec1fcd53100f01fd1181fdf4Jens Axboe 745cfbcd0dc2fcbf0f9bfe92ff8f4ab17f21e46d21eJens Axboe hdr_inc = get_hdr_inc(td, io_u); 746e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 747a12a3b4d8903b9941705067442c2c530a94635f3Jens Axboe ret = 0; 7485ec10eaad3b09875b91e19a20bbdfa06f2117562Jens Axboe for (p = io_u->buf; p < io_u->buf + io_u->buflen; 7495ec10eaad3b09875b91e19a20bbdfa06f2117562Jens Axboe p += hdr_inc, hdr_num++) { 750bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe struct vcont vc = { 751bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe .io_u = io_u, 752bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe .hdr_num = hdr_num, 7537d9fb455aadc0c0363489591775496f27f4a560aJens Axboe .td = td, 754bfacf389e9edb1b63ca0b9c6cb2aea8b06d53aceJens Axboe }; 755f00b210f73a8562c2cc64d73aee04875d25162b0Jens Axboe unsigned int verify_type; 756936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe 757f3e6cb95fecf867ec46e3dfd59ef1a1db1b6ad2aJens Axboe if (ret && td->o.verify_fatal) 758a12a3b4d8903b9941705067442c2c530a94635f3Jens Axboe break; 759f3e6cb95fecf867ec46e3dfd59ef1a1db1b6ad2aJens Axboe 7602b13e716c0921356c0930522718e00b8df34293aJens Axboe header_size = __hdr_size(td->o.verify); 761a59e170d683f9ccbaa45648d7ae9a13b70e3822cJens Axboe if (td->o.verify_offset) 7622b13e716c0921356c0930522718e00b8df34293aJens Axboe memswp(p, p + td->o.verify_offset, header_size); 76395646108f83a34c0486d7f3ed19e99c1d93691e6Jens Axboe hdr = p; 764e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 765c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat /* 766c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat * Make rand_seed check pass when have verifysort or 767c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat * verify_backlog. 768c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat */ 769c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat if (td->o.verifysort || (td->flags & TD_F_VER_BACKLOG)) 770c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat io_u->rand_seed = hdr->rand_seed; 771c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat 772e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse ret = verify_header(io_u, hdr); 773e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse switch (ret) { 774e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse case 0: 775e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse break; 776e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse case 1: 777ae38c0dcd570265aef09aa132ce27278d8909a1dJens Axboe log_err("verify: bad magic header %x, wanted %x at " 778ae38c0dcd570265aef09aa132ce27278d8909a1dJens Axboe "file %s offset %llu, length %u\n", 779d3a173a927084660c365d6aa162016ebb0746946Jens Axboe hdr->magic, FIO_HDR_MAGIC, 780c9b440312ff63c2b4c8548530f24c2eb5820b6a3Jens Axboe io_u->file->file_name, 781c9b440312ff63c2b4c8548530f24c2eb5820b6a3Jens Axboe io_u->offset + hdr_num * hdr->len, hdr->len); 7829fd189697dbf9fb7878093be5e41f25b7f932c2dJens Axboe return EILSEQ; 783e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse break; 784e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse case 2: 785e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse log_err("fio: verify header exceeds buffer length (%u " 786e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse "> %lu)\n", hdr->len, io_u->buflen); 787e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse return EILSEQ; 788e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse break; 789e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse case 3: 790e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse log_err("verify: bad header rand_seed %"PRIu64 791e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse ", wanted %"PRIu64" at file %s offset %llu, " 792e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse "length %u\n", 793e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse hdr->rand_seed, io_u->rand_seed, 794e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse io_u->file->file_name, 795e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse io_u->offset + hdr_num * hdr->len, hdr->len); 796e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse return EILSEQ; 797e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse break; 798e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse case 4: 799e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse return EILSEQ; 800e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse break; 801e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse default: 802e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse log_err("verify: unknown header error at file %s " 803e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse "offset %llu, length %u\n", 804e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse io_u->file->file_name, 805e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse io_u->offset + hdr_num * hdr->len, hdr->len); 806e9e177cce9df45eca77a4a0822a8247e60f2680bJuan Casse return EILSEQ; 8073f199b01fc9cc7a0f4915db266d7a28bc46e84f0Jens Axboe } 8083f199b01fc9cc7a0f4915db266d7a28bc46e84f0Jens Axboe 809f00b210f73a8562c2cc64d73aee04875d25162b0Jens Axboe if (td->o.verify != VERIFY_NONE) 810f00b210f73a8562c2cc64d73aee04875d25162b0Jens Axboe verify_type = td->o.verify; 811f00b210f73a8562c2cc64d73aee04875d25162b0Jens Axboe else 812f00b210f73a8562c2cc64d73aee04875d25162b0Jens Axboe verify_type = hdr->verify_type; 813f00b210f73a8562c2cc64d73aee04875d25162b0Jens Axboe 814f00b210f73a8562c2cc64d73aee04875d25162b0Jens Axboe switch (verify_type) { 8153f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis case VERIFY_MD5: 816936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe ret = verify_io_u_md5(hdr, &vc); 8173f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis break; 8183f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis case VERIFY_CRC64: 819936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe ret = verify_io_u_crc64(hdr, &vc); 8203f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis break; 821bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe case VERIFY_CRC32C: 8223845591fadea480177223e28c9d1c03642d34f0eJens Axboe case VERIFY_CRC32C_INTEL: 823936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe ret = verify_io_u_crc32c(hdr, &vc); 824bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe break; 8253f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis case VERIFY_CRC32: 826936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe ret = verify_io_u_crc32(hdr, &vc); 8273f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis break; 8283f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis case VERIFY_CRC16: 829936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe ret = verify_io_u_crc16(hdr, &vc); 8303f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis break; 8313f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis case VERIFY_CRC7: 832936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe ret = verify_io_u_crc7(hdr, &vc); 8333f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis break; 834cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe case VERIFY_SHA256: 835936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe ret = verify_io_u_sha256(hdr, &vc); 836cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe break; 837cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe case VERIFY_SHA512: 838936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe ret = verify_io_u_sha512(hdr, &vc); 839cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe break; 840844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe case VERIFY_XXHASH: 841844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe ret = verify_io_u_xxhash(hdr, &vc); 842844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe break; 8437437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis case VERIFY_META: 84492bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe ret = verify_io_u_meta(hdr, &vc); 8457437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis break; 8467c353ceb3257b132f4c98326046b42201e070731Jens Axboe case VERIFY_SHA1: 847936216f8cd8ae2261282453d82a2a8232bb4db45Jens Axboe ret = verify_io_u_sha1(hdr, &vc); 8487c353ceb3257b132f4c98326046b42201e070731Jens Axboe break; 84992bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe case VERIFY_PATTERN: 85092bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe ret = verify_io_u_pattern(hdr, &vc); 85192bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe break; 8523f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis default: 8533f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis log_err("Bad verify type %u\n", hdr->verify_type); 854d16d4e09352f618c266d1efe27cbb25a95df6e86Jens Axboe ret = EINVAL; 8553f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis } 856f00b210f73a8562c2cc64d73aee04875d25162b0Jens Axboe 857f00b210f73a8562c2cc64d73aee04875d25162b0Jens Axboe if (ret && verify_type != hdr->verify_type) 858f00b210f73a8562c2cc64d73aee04875d25162b0Jens Axboe log_err("fio: verify type mismatch (%u media, %u given)\n", 859f00b210f73a8562c2cc64d73aee04875d25162b0Jens Axboe hdr->verify_type, verify_type); 8603f9f4e2631c4f5d33f125dd71db20c1773d22c91Shawn Lewis } 861a7dfe862c2e3fe72b8db6a6e67302048d3731c6dJens Axboe 8620d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboedone: 863f3e6cb95fecf867ec46e3dfd59ef1a1db1b6ad2aJens Axboe if (ret && td->o.verify_fatal) 864f3e6cb95fecf867ec46e3dfd59ef1a1db1b6ad2aJens Axboe td->terminate = 1; 865f3e6cb95fecf867ec46e3dfd59ef1a1db1b6ad2aJens Axboe 866a12a3b4d8903b9941705067442c2c530a94635f3Jens Axboe return ret; 867e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe} 868e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 8697437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewisstatic void fill_meta(struct verify_header *hdr, struct thread_data *td, 8705ec10eaad3b09875b91e19a20bbdfa06f2117562Jens Axboe struct io_u *io_u, unsigned int header_num) 8717437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis{ 8727437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis struct vhdr_meta *vh = hdr_priv(hdr); 8737437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis 8747437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis vh->thread = td->thread_number; 8757437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis 8767437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis vh->time_sec = io_u->start_time.tv_sec; 8777437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis vh->time_usec = io_u->start_time.tv_usec; 8787437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis 879da0a7bd224bb9331f27bb4b20394dd5c8fa3acb0Juan Casse vh->numberio = io_u->numberio; 8807437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis 8817437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis vh->offset = io_u->offset + header_num * td->o.verify_interval; 8827437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis} 8837437ee87e4f6457168e8440f9308b4296ddb9263Shawn Lewis 884844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboestatic void fill_xxhash(struct verify_header *hdr, void *p, unsigned int len) 885844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe{ 886844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe struct vhdr_xxhash *vh = hdr_priv(hdr); 887844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe void *state; 888844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe 889844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe state = XXH32_init(1); 890844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe XXH32_update(state, p, len); 891844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe vh->hash = XXH32_digest(state); 892844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe} 893844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe 894cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboestatic void fill_sha512(struct verify_header *hdr, void *p, unsigned int len) 895cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe{ 896546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_sha512 *vh = hdr_priv(hdr); 89725dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe struct fio_sha512_ctx sha512_ctx = { 898546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe .buf = vh->sha512, 899cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe }; 900cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 90125dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe fio_sha512_init(&sha512_ctx); 90225dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe fio_sha512_update(&sha512_ctx, p, len); 903cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe} 904cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 905cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboestatic void fill_sha256(struct verify_header *hdr, void *p, unsigned int len) 906cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe{ 907546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_sha256 *vh = hdr_priv(hdr); 90825dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe struct fio_sha256_ctx sha256_ctx = { 909546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe .buf = vh->sha256, 910cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe }; 911cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 91225dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe fio_sha256_init(&sha256_ctx); 91325dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe fio_sha256_update(&sha256_ctx, p, len); 914cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe} 915cd14cc1079771ece34fce5b30d47c7b273ee56e5Jens Axboe 9167c353ceb3257b132f4c98326046b42201e070731Jens Axboestatic void fill_sha1(struct verify_header *hdr, void *p, unsigned int len) 9177c353ceb3257b132f4c98326046b42201e070731Jens Axboe{ 9187c353ceb3257b132f4c98326046b42201e070731Jens Axboe struct vhdr_sha1 *vh = hdr_priv(hdr); 91925dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe struct fio_sha1_ctx sha1_ctx = { 9207c353ceb3257b132f4c98326046b42201e070731Jens Axboe .H = vh->sha1, 9217c353ceb3257b132f4c98326046b42201e070731Jens Axboe }; 9227c353ceb3257b132f4c98326046b42201e070731Jens Axboe 92325dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe fio_sha1_init(&sha1_ctx); 92425dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe fio_sha1_update(&sha1_ctx, p, len); 9257c353ceb3257b132f4c98326046b42201e070731Jens Axboe} 9267c353ceb3257b132f4c98326046b42201e070731Jens Axboe 9271e154bdbf14a1d352117aea057035235b66f0381Jens Axboestatic void fill_crc7(struct verify_header *hdr, void *p, unsigned int len) 9281e154bdbf14a1d352117aea057035235b66f0381Jens Axboe{ 929546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_crc7 *vh = hdr_priv(hdr); 930546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe 93125dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe vh->crc7 = fio_crc7(p, len); 9321e154bdbf14a1d352117aea057035235b66f0381Jens Axboe} 9331e154bdbf14a1d352117aea057035235b66f0381Jens Axboe 934969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboestatic void fill_crc16(struct verify_header *hdr, void *p, unsigned int len) 935969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboe{ 936546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_crc16 *vh = hdr_priv(hdr); 937546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe 93825dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe vh->crc16 = fio_crc16(p, len); 939969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboe} 940969f7ed32353ade93ea30542a4993b75b94e3f8aJens Axboe 941e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboestatic void fill_crc32(struct verify_header *hdr, void *p, unsigned int len) 942e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe{ 943546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_crc32 *vh = hdr_priv(hdr); 944546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe 94525dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe vh->crc32 = fio_crc32(p, len); 946e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe} 947e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 948bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboestatic void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len) 949bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe{ 950bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe struct vhdr_crc32 *vh = hdr_priv(hdr); 951bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe 95225dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe vh->crc32 = fio_crc32c(p, len); 953bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe} 954bac39e0e8807a1d52863ad8304e67221df2bc63bJens Axboe 955d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboestatic void fill_crc64(struct verify_header *hdr, void *p, unsigned int len) 956d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboe{ 957546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_crc64 *vh = hdr_priv(hdr); 958546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe 95925dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe vh->crc64 = fio_crc64(p, len); 960d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboe} 961d77a7af3e718200bbd77d76194725ead4bdb3591Jens Axboe 962e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboestatic void fill_md5(struct verify_header *hdr, void *p, unsigned int len) 963e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe{ 964546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe struct vhdr_md5 *vh = hdr_priv(hdr); 96525dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe struct fio_md5_ctx md5_ctx = { 966546dfd9ff99eec4af5d1c6c2ca702b7cf9656334Jens Axboe .hash = (uint32_t *) vh->md5_digest, 9678c432325c3df2075a77b27eab8a87704cf7b48eeJens Axboe }; 968e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 96925dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe fio_md5_init(&md5_ctx); 97025dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe fio_md5_update(&md5_ctx, p, len); 971e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe} 972e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 9737d9fb455aadc0c0363489591775496f27f4a560aJens Axboestatic void populate_hdr(struct thread_data *td, struct io_u *io_u, 9747d9fb455aadc0c0363489591775496f27f4a560aJens Axboe struct verify_header *hdr, unsigned int header_num, 9757d9fb455aadc0c0363489591775496f27f4a560aJens Axboe unsigned int header_len) 9767d9fb455aadc0c0363489591775496f27f4a560aJens Axboe{ 9777d9fb455aadc0c0363489591775496f27f4a560aJens Axboe unsigned int data_len; 9787d9fb455aadc0c0363489591775496f27f4a560aJens Axboe void *data, *p; 9797d9fb455aadc0c0363489591775496f27f4a560aJens Axboe 9807d9fb455aadc0c0363489591775496f27f4a560aJens Axboe p = (void *) hdr; 9817d9fb455aadc0c0363489591775496f27f4a560aJens Axboe 982d3a173a927084660c365d6aa162016ebb0746946Jens Axboe hdr->magic = FIO_HDR_MAGIC; 9837d9fb455aadc0c0363489591775496f27f4a560aJens Axboe hdr->verify_type = td->o.verify; 984d3a173a927084660c365d6aa162016ebb0746946Jens Axboe hdr->len = header_len; 9857d9fb455aadc0c0363489591775496f27f4a560aJens Axboe hdr->rand_seed = io_u->rand_seed; 98625dfa848abbb6c35b4d45fabd5a8e82cb77fb285Jens Axboe hdr->crc32 = fio_crc32c(p, offsetof(struct verify_header, crc32)); 987f65d1c2663ac6007eac1a2063fe25b4275d013e4Jens Axboe 9887d9fb455aadc0c0363489591775496f27f4a560aJens Axboe data_len = header_len - hdr_size(hdr); 9897d9fb455aadc0c0363489591775496f27f4a560aJens Axboe 9907d9fb455aadc0c0363489591775496f27f4a560aJens Axboe data = p + hdr_size(hdr); 9917d9fb455aadc0c0363489591775496f27f4a560aJens Axboe switch (td->o.verify) { 9927d9fb455aadc0c0363489591775496f27f4a560aJens Axboe case VERIFY_MD5: 9937d9fb455aadc0c0363489591775496f27f4a560aJens Axboe dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n", 9947d9fb455aadc0c0363489591775496f27f4a560aJens Axboe io_u, hdr->len); 9957d9fb455aadc0c0363489591775496f27f4a560aJens Axboe fill_md5(hdr, data, data_len); 9967d9fb455aadc0c0363489591775496f27f4a560aJens Axboe break; 9977d9fb455aadc0c0363489591775496f27f4a560aJens Axboe case VERIFY_CRC64: 9987d9fb455aadc0c0363489591775496f27f4a560aJens Axboe dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n", 9997d9fb455aadc0c0363489591775496f27f4a560aJens Axboe io_u, hdr->len); 10007d9fb455aadc0c0363489591775496f27f4a560aJens Axboe fill_crc64(hdr, data, data_len); 10017d9fb455aadc0c0363489591775496f27f4a560aJens Axboe break; 10027d9fb455aadc0c0363489591775496f27f4a560aJens Axboe case VERIFY_CRC32C: 10037d9fb455aadc0c0363489591775496f27f4a560aJens Axboe case VERIFY_CRC32C_INTEL: 10047d9fb455aadc0c0363489591775496f27f4a560aJens Axboe dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n", 10057d9fb455aadc0c0363489591775496f27f4a560aJens Axboe io_u, hdr->len); 10067d9fb455aadc0c0363489591775496f27f4a560aJens Axboe fill_crc32c(hdr, data, data_len); 10077d9fb455aadc0c0363489591775496f27f4a560aJens Axboe break; 10087d9fb455aadc0c0363489591775496f27f4a560aJens Axboe case VERIFY_CRC32: 10097d9fb455aadc0c0363489591775496f27f4a560aJens Axboe dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n", 10107d9fb455aadc0c0363489591775496f27f4a560aJens Axboe io_u, hdr->len); 10117d9fb455aadc0c0363489591775496f27f4a560aJens Axboe fill_crc32(hdr, data, data_len); 10127d9fb455aadc0c0363489591775496f27f4a560aJens Axboe break; 10137d9fb455aadc0c0363489591775496f27f4a560aJens Axboe case VERIFY_CRC16: 10147d9fb455aadc0c0363489591775496f27f4a560aJens Axboe dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n", 10157d9fb455aadc0c0363489591775496f27f4a560aJens Axboe io_u, hdr->len); 10167d9fb455aadc0c0363489591775496f27f4a560aJens Axboe fill_crc16(hdr, data, data_len); 10177d9fb455aadc0c0363489591775496f27f4a560aJens Axboe break; 10187d9fb455aadc0c0363489591775496f27f4a560aJens Axboe case VERIFY_CRC7: 10197d9fb455aadc0c0363489591775496f27f4a560aJens Axboe dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n", 10207d9fb455aadc0c0363489591775496f27f4a560aJens Axboe io_u, hdr->len); 10217d9fb455aadc0c0363489591775496f27f4a560aJens Axboe fill_crc7(hdr, data, data_len); 10227d9fb455aadc0c0363489591775496f27f4a560aJens Axboe break; 10237d9fb455aadc0c0363489591775496f27f4a560aJens Axboe case VERIFY_SHA256: 10247d9fb455aadc0c0363489591775496f27f4a560aJens Axboe dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n", 10257d9fb455aadc0c0363489591775496f27f4a560aJens Axboe io_u, hdr->len); 10267d9fb455aadc0c0363489591775496f27f4a560aJens Axboe fill_sha256(hdr, data, data_len); 10277d9fb455aadc0c0363489591775496f27f4a560aJens Axboe break; 10287d9fb455aadc0c0363489591775496f27f4a560aJens Axboe case VERIFY_SHA512: 10297d9fb455aadc0c0363489591775496f27f4a560aJens Axboe dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n", 10307d9fb455aadc0c0363489591775496f27f4a560aJens Axboe io_u, hdr->len); 10317d9fb455aadc0c0363489591775496f27f4a560aJens Axboe fill_sha512(hdr, data, data_len); 10327d9fb455aadc0c0363489591775496f27f4a560aJens Axboe break; 1033844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe case VERIFY_XXHASH: 1034844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe dprint(FD_VERIFY, "fill xxhash io_u %p, len %u\n", 1035844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe io_u, hdr->len); 1036844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe fill_xxhash(hdr, data, data_len); 1037844ea60273f4e7862150d9c6430cdb15768e2f34Jens Axboe break; 10387d9fb455aadc0c0363489591775496f27f4a560aJens Axboe case VERIFY_META: 10397d9fb455aadc0c0363489591775496f27f4a560aJens Axboe dprint(FD_VERIFY, "fill meta io_u %p, len %u\n", 10407d9fb455aadc0c0363489591775496f27f4a560aJens Axboe io_u, hdr->len); 10417d9fb455aadc0c0363489591775496f27f4a560aJens Axboe fill_meta(hdr, td, io_u, header_num); 10427d9fb455aadc0c0363489591775496f27f4a560aJens Axboe break; 10437d9fb455aadc0c0363489591775496f27f4a560aJens Axboe case VERIFY_SHA1: 10447d9fb455aadc0c0363489591775496f27f4a560aJens Axboe dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n", 10457d9fb455aadc0c0363489591775496f27f4a560aJens Axboe io_u, hdr->len); 10467d9fb455aadc0c0363489591775496f27f4a560aJens Axboe fill_sha1(hdr, data, data_len); 10477d9fb455aadc0c0363489591775496f27f4a560aJens Axboe break; 104892bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe case VERIFY_PATTERN: 104992bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe /* nothing to do here */ 105092bf48d530ab09bae4058df711c4723c73da51b9Jens Axboe break; 10517d9fb455aadc0c0363489591775496f27f4a560aJens Axboe default: 10527d9fb455aadc0c0363489591775496f27f4a560aJens Axboe log_err("fio: bad verify type: %d\n", td->o.verify); 10537d9fb455aadc0c0363489591775496f27f4a560aJens Axboe assert(0); 10547d9fb455aadc0c0363489591775496f27f4a560aJens Axboe } 10557d9fb455aadc0c0363489591775496f27f4a560aJens Axboe if (td->o.verify_offset) 10567d9fb455aadc0c0363489591775496f27f4a560aJens Axboe memswp(p, p + td->o.verify_offset, hdr_size(hdr)); 10577d9fb455aadc0c0363489591775496f27f4a560aJens Axboe} 10587d9fb455aadc0c0363489591775496f27f4a560aJens Axboe 1059e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe/* 1060e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe * fill body of io_u->buf with random data and add a header with the 1061c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe * checksum of choice 1062e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe */ 1063e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboevoid populate_verify_io_u(struct thread_data *td, struct io_u *io_u) 1064e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe{ 10659cc3d1506b9f7ce0a9dba38c32c418e54ea0b573Jens Axboe if (td->o.verify == VERIFY_NULL) 10669cc3d1506b9f7ce0a9dba38c32c418e54ea0b573Jens Axboe return; 10679cc3d1506b9f7ce0a9dba38c32c418e54ea0b573Jens Axboe 1068da0a7bd224bb9331f27bb4b20394dd5c8fa3acb0Juan Casse io_u->numberio = td->io_issues[io_u->ddir]; 1069da0a7bd224bb9331f27bb4b20394dd5c8fa3acb0Juan Casse 1070c50ca7bd6d5b84e24e9ddcdb1765fb4070804503Jens Axboe fill_pattern_headers(td, io_u, 0, 0); 1071e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe} 1072e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 1073e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboeint get_next_verify(struct thread_data *td, struct io_u *io_u) 1074e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe{ 10758de8f047bd025f12d23cfc3fc1793434c6d8ff94Jens Axboe struct io_piece *ipo = NULL; 1076e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 1077d2d7fa53c18f1fc3fb89f3fe20df9b39986bde72Jens Axboe /* 1078d2d7fa53c18f1fc3fb89f3fe20df9b39986bde72Jens Axboe * this io_u is from a requeue, we already filled the offsets 1079d2d7fa53c18f1fc3fb89f3fe20df9b39986bde72Jens Axboe */ 1080d2d7fa53c18f1fc3fb89f3fe20df9b39986bde72Jens Axboe if (io_u->file) 1081d2d7fa53c18f1fc3fb89f3fe20df9b39986bde72Jens Axboe return 0; 1082d2d7fa53c18f1fc3fb89f3fe20df9b39986bde72Jens Axboe 10838de8f047bd025f12d23cfc3fc1793434c6d8ff94Jens Axboe if (!RB_EMPTY_ROOT(&td->io_hist_tree)) { 10848de8f047bd025f12d23cfc3fc1793434c6d8ff94Jens Axboe struct rb_node *n = rb_first(&td->io_hist_tree); 1085e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 10868de8f047bd025f12d23cfc3fc1793434c6d8ff94Jens Axboe ipo = rb_entry(n, struct io_piece, rb_node); 1087f940128526dbe468a1951cce10c2fe5dbd23875fJens Axboe 1088f940128526dbe468a1951cce10c2fe5dbd23875fJens Axboe /* 1089f940128526dbe468a1951cce10c2fe5dbd23875fJens Axboe * Ensure that the associated IO has completed 1090f940128526dbe468a1951cce10c2fe5dbd23875fJens Axboe */ 1091f940128526dbe468a1951cce10c2fe5dbd23875fJens Axboe read_barrier(); 1092f940128526dbe468a1951cce10c2fe5dbd23875fJens Axboe if (ipo->flags & IP_F_IN_FLIGHT) 1093f940128526dbe468a1951cce10c2fe5dbd23875fJens Axboe goto nothing; 1094f940128526dbe468a1951cce10c2fe5dbd23875fJens Axboe 10954b87898e8d76aaf05baec83077a11311c1447397Jens Axboe rb_erase(n, &td->io_hist_tree); 1096a917a8b3dfeefdd7007ba2f46f21fc145574309dJens Axboe assert(ipo->flags & IP_F_ONRB); 1097a917a8b3dfeefdd7007ba2f46f21fc145574309dJens Axboe ipo->flags &= ~IP_F_ONRB; 109801743ee1718e7ec4b16ae3e53c8f64900c6052ccJens Axboe } else if (!flist_empty(&td->io_hist_list)) { 109901743ee1718e7ec4b16ae3e53c8f64900c6052ccJens Axboe ipo = flist_entry(td->io_hist_list.next, struct io_piece, list); 1100f940128526dbe468a1951cce10c2fe5dbd23875fJens Axboe 1101f940128526dbe468a1951cce10c2fe5dbd23875fJens Axboe /* 1102f940128526dbe468a1951cce10c2fe5dbd23875fJens Axboe * Ensure that the associated IO has completed 1103f940128526dbe468a1951cce10c2fe5dbd23875fJens Axboe */ 1104f940128526dbe468a1951cce10c2fe5dbd23875fJens Axboe read_barrier(); 1105f940128526dbe468a1951cce10c2fe5dbd23875fJens Axboe if (ipo->flags & IP_F_IN_FLIGHT) 1106f940128526dbe468a1951cce10c2fe5dbd23875fJens Axboe goto nothing; 1107f940128526dbe468a1951cce10c2fe5dbd23875fJens Axboe 110801743ee1718e7ec4b16ae3e53c8f64900c6052ccJens Axboe flist_del(&ipo->list); 1109a917a8b3dfeefdd7007ba2f46f21fc145574309dJens Axboe assert(ipo->flags & IP_F_ONLIST); 1110a917a8b3dfeefdd7007ba2f46f21fc145574309dJens Axboe ipo->flags &= ~IP_F_ONLIST; 11118de8f047bd025f12d23cfc3fc1793434c6d8ff94Jens Axboe } 1112e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 11138de8f047bd025f12d23cfc3fc1793434c6d8ff94Jens Axboe if (ipo) { 11140d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe td->io_hist_len--; 11150d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe 1116e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe io_u->offset = ipo->offset; 1117e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe io_u->buflen = ipo->len; 1118da0a7bd224bb9331f27bb4b20394dd5c8fa3acb0Juan Casse io_u->numberio = ipo->numberio; 111936167d82e5f49dee91c6d2cd426068edee90e36fJens Axboe io_u->file = ipo->file; 112082af2a7ca1a543b41c003de69d5e3c36860f47d5Jens Axboe io_u->flags |= IO_U_F_VER_LIST; 112197af62cec418cd722fb43c6010d6430534e12353Jens Axboe 11220d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe if (ipo->flags & IP_F_TRIMMED) 11230d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe io_u->flags |= IO_U_F_TRIMMED; 11240d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe 1125d6aed795f2e3e403828abf60874dd2d6e8342a1bJens Axboe if (!fio_file_open(io_u->file)) { 112697af62cec418cd722fb43c6010d6430534e12353Jens Axboe int r = td_io_open_file(td, io_u->file); 112797af62cec418cd722fb43c6010d6430534e12353Jens Axboe 1128bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe if (r) { 1129bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "failed file %s open\n", 1130bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe io_u->file->file_name); 113197af62cec418cd722fb43c6010d6430534e12353Jens Axboe return 1; 1132bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe } 113397af62cec418cd722fb43c6010d6430534e12353Jens Axboe } 113497af62cec418cd722fb43c6010d6430534e12353Jens Axboe 113597af62cec418cd722fb43c6010d6430534e12353Jens Axboe get_file(ipo->file); 1136d6aed795f2e3e403828abf60874dd2d6e8342a1bJens Axboe assert(fio_file_open(io_u->file)); 1137e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe io_u->ddir = DDIR_READ; 113836167d82e5f49dee91c6d2cd426068edee90e36fJens Axboe io_u->xfer_buf = io_u->buf; 113936167d82e5f49dee91c6d2cd426068edee90e36fJens Axboe io_u->xfer_buflen = io_u->buflen; 11400d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe 11410d29de831183dfd049c97a03008d425ce21e2fa4Jens Axboe remove_trim_entry(td, ipo); 1142e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe free(ipo); 1143bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u); 1144c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat 1145c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat if (!td->o.verify_pattern_bytes) { 1146c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat io_u->rand_seed = __rand(&td->__verify_state); 1147c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat if (sizeof(int) != sizeof(long *)) 1148c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat io_u->rand_seed *= __rand(&td->__verify_state); 1149c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat } 1150e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe return 0; 1151e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe } 1152e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe 1153f940128526dbe468a1951cce10c2fe5dbd23875fJens Axboenothing: 1154bd6f78b2f0b6375383948f126368804612ec0035Jens Axboe dprint(FD_VERIFY, "get_next_verify: empty\n"); 1155e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe return 1; 1156e29d1b70a21e29801fb35dfbc1b236b7c8514055Jens Axboe} 1157e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 1158dc5bfbb227f1af83adf0b686d147b7ae2107c144Jens Axboevoid fio_verify_init(struct thread_data *td) 1159dc5bfbb227f1af83adf0b686d147b7ae2107c144Jens Axboe{ 1160dc5bfbb227f1af83adf0b686d147b7ae2107c144Jens Axboe if (td->o.verify == VERIFY_CRC32C_INTEL || 1161dc5bfbb227f1af83adf0b686d147b7ae2107c144Jens Axboe td->o.verify == VERIFY_CRC32C) { 1162dc5bfbb227f1af83adf0b686d147b7ae2107c144Jens Axboe crc32c_intel_probe(); 1163dc5bfbb227f1af83adf0b686d147b7ae2107c144Jens Axboe } 1164dc5bfbb227f1af83adf0b686d147b7ae2107c144Jens Axboe} 1165dc5bfbb227f1af83adf0b686d147b7ae2107c144Jens Axboe 1166e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboestatic void *verify_async_thread(void *data) 1167e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe{ 1168e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe struct thread_data *td = data; 1169e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe struct io_u *io_u; 1170e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe int ret = 0; 1171e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 1172e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe if (td->o.verify_cpumask_set && 1173e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe fio_setaffinity(td->pid, td->o.verify_cpumask)) { 1174e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe log_err("fio: failed setting verify thread affinity\n"); 1175e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe goto done; 1176e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe } 1177e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 1178e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe do { 1179e53ab27c7425241d64e8d64ad77850a776c01375Jens Axboe FLIST_HEAD(list); 1180e53ab27c7425241d64e8d64ad77850a776c01375Jens Axboe 1181e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe read_barrier(); 1182e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe if (td->verify_thread_exit) 1183e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe break; 1184e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 1185e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_mutex_lock(&td->io_u_lock); 1186e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 1187e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe while (flist_empty(&td->verify_list) && 1188e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe !td->verify_thread_exit) { 1189b36e298b5bdfc31ffc9a4c14e94a9c5c748888b0Jens Axboe ret = pthread_cond_wait(&td->verify_cond, 1190b36e298b5bdfc31ffc9a4c14e94a9c5c748888b0Jens Axboe &td->io_u_lock); 1191e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe if (ret) { 1192e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_mutex_unlock(&td->io_u_lock); 1193e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe break; 1194e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe } 1195e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe } 1196e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 1197e53ab27c7425241d64e8d64ad77850a776c01375Jens Axboe flist_splice_init(&td->verify_list, &list); 1198e53ab27c7425241d64e8d64ad77850a776c01375Jens Axboe pthread_mutex_unlock(&td->io_u_lock); 1199e53ab27c7425241d64e8d64ad77850a776c01375Jens Axboe 1200e53ab27c7425241d64e8d64ad77850a776c01375Jens Axboe if (flist_empty(&list)) 1201e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe continue; 1202e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 1203e53ab27c7425241d64e8d64ad77850a776c01375Jens Axboe while (!flist_empty(&list)) { 12042ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe io_u = flist_entry(list.next, struct io_u, verify_list); 12052ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe flist_del(&io_u->verify_list); 1206e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 1207d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe ret = verify_io_u(td, io_u); 1208e53ab27c7425241d64e8d64ad77850a776c01375Jens Axboe put_io_u(td, io_u); 1209d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe if (!ret) 1210d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe continue; 12118b28bd41375930664a0ff9ff9b101a88ac416ac5Dmitry Monakhov if (td_non_fatal_error(td, ERROR_TYPE_VERIFY_BIT, ret)) { 1212d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe update_error_count(td, ret); 1213d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe td_clear_error(td); 1214d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe ret = 0; 1215d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe } 1216e53ab27c7425241d64e8d64ad77850a776c01375Jens Axboe } 1217e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe } while (!ret); 1218e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 1219d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe if (ret) { 1220d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe td_verror(td, ret, "async_verify"); 1221f3e6cb95fecf867ec46e3dfd59ef1a1db1b6ad2aJens Axboe if (td->o.verify_fatal) 1222f3e6cb95fecf867ec46e3dfd59ef1a1db1b6ad2aJens Axboe td->terminate = 1; 1223d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe } 1224d561f2ab37ef5729882ef3c6e6ed8ee49414955fJens Axboe 1225e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboedone: 1226e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_mutex_lock(&td->io_u_lock); 1227e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe td->nr_verify_threads--; 1228e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_mutex_unlock(&td->io_u_lock); 1229e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 1230e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_cond_signal(&td->free_cond); 1231e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe return NULL; 1232e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe} 1233e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 1234e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboeint verify_async_init(struct thread_data *td) 1235e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe{ 1236e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe int i, ret; 1237304a47c7d94f407cc72a87025679a67f02288447bart Van Assche pthread_attr_t attr; 1238304a47c7d94f407cc72a87025679a67f02288447bart Van Assche 1239304a47c7d94f407cc72a87025679a67f02288447bart Van Assche pthread_attr_init(&attr); 1240304a47c7d94f407cc72a87025679a67f02288447bart Van Assche pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN); 1241e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 1242e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe td->verify_thread_exit = 0; 1243e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 1244e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async); 1245e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe for (i = 0; i < td->o.verify_async; i++) { 1246304a47c7d94f407cc72a87025679a67f02288447bart Van Assche ret = pthread_create(&td->verify_threads[i], &attr, 1247e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe verify_async_thread, td); 1248e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe if (ret) { 1249e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe log_err("fio: async verify creation failed: %s\n", 1250e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe strerror(ret)); 1251e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe break; 1252e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe } 1253e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe ret = pthread_detach(td->verify_threads[i]); 1254e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe if (ret) { 1255e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe log_err("fio: async verify thread detach failed: %s\n", 1256e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe strerror(ret)); 1257e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe break; 1258e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe } 1259e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe td->nr_verify_threads++; 1260e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe } 1261e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 1262304a47c7d94f407cc72a87025679a67f02288447bart Van Assche pthread_attr_destroy(&attr); 1263304a47c7d94f407cc72a87025679a67f02288447bart Van Assche 1264e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe if (i != td->o.verify_async) { 1265e40823b10920f679d84f3f2222e319c168b1e7daJens Axboe log_err("fio: only %d verify threads started, exiting\n", i); 1266e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe td->verify_thread_exit = 1; 1267e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe write_barrier(); 1268e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_cond_broadcast(&td->verify_cond); 1269e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe return 1; 1270e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe } 1271e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 1272e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe return 0; 1273e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe} 1274e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 1275e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboevoid verify_async_exit(struct thread_data *td) 1276e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe{ 1277e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe td->verify_thread_exit = 1; 1278e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe write_barrier(); 1279e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_cond_broadcast(&td->verify_cond); 1280e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 1281e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_mutex_lock(&td->io_u_lock); 1282e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 1283e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe while (td->nr_verify_threads) 1284e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_cond_wait(&td->free_cond, &td->io_u_lock); 1285e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe 1286e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe pthread_mutex_unlock(&td->io_u_lock); 1287e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe free(td->verify_threads); 1288e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe td->verify_threads = NULL; 1289e8462bd8250cf3ff2d41f17e1a4d4cefc70b6b37Jens Axboe} 1290