12e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe/* 22e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * fio - the flexible io tester 32e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * 42e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * Copyright (C) 2005 Jens Axboe <axboe@suse.de> 52e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * Copyright (C) 2006-2012 Jens Axboe <axboe@kernel.dk> 62e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * 72e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * The license below covers all files distributed with fio unless otherwise 82e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * noted in the file itself. 92e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * 102e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * This program is free software; you can redistribute it and/or modify 112e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * it under the terms of the GNU General Public License version 2 as 122e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * published by the Free Software Foundation. 132e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * 142e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * This program is distributed in the hope that it will be useful, 152e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * but WITHOUT ANY WARRANTY; without even the implied warranty of 162e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 172e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * GNU General Public License for more details. 182e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * 192e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * You should have received a copy of the GNU General Public License 202e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * along with this program; if not, write to the Free Software 212e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 222e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * 232e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 242e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include <unistd.h> 252e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include <fcntl.h> 262e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include <string.h> 272e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include <limits.h> 282e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include <signal.h> 292e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include <time.h> 302e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include <locale.h> 312e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include <assert.h> 322e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include <time.h> 33e43606c2b13ad7fc1af2bbe4a61cf8480ee3a532Bruce Cran#include <inttypes.h> 342e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include <sys/stat.h> 352e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include <sys/wait.h> 362e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include <sys/ipc.h> 372e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include <sys/mman.h> 382e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 392e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include "fio.h" 40a5e0ee11ec34cfc70a78aee4f7a2eeb01d9cf41fOleg#ifndef FIO_NO_HAVE_SHM_H 41a5e0ee11ec34cfc70a78aee4f7a2eeb01d9cf41fOleg#include <sys/shm.h> 42a5e0ee11ec34cfc70a78aee4f7a2eeb01d9cf41fOleg#endif 432e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include "hash.h" 442e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include "smalloc.h" 452e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include "verify.h" 462e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include "trim.h" 472e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include "diskutil.h" 482e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include "cgroup.h" 492e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include "profile.h" 502e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include "lib/rand.h" 512e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include "memalign.h" 522e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#include "server.h" 5344404c5a7f90aa42c3228b56b6f686f15a50fb29Jens Axboe#include "lib/getrusage.h" 54f2a2ce0eedb44eaa8689e4cbfa77c79b1751b216Huadong Liu#include "idletime.h" 55002fe73409d1e3d5e7dfe2885f75885bfaf506bcJens Axboe#include "err.h" 562e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 572e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic pthread_t disk_util_thread; 589ec7779f89d4dde9fe1e23d68abaefda54b35de7Jens Axboestatic struct fio_mutex *disk_thread_mutex; 592e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic struct fio_mutex *startup_mutex; 602e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic struct flist_head *cgroup_list; 612e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic char *cgroup_mnt; 622e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic int exit_value; 632e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic volatile int fio_abort; 643a5f6bde1d14faa9e60cde1c8b45c101bc462f34Jens Axboestatic unsigned int nr_process = 0; 653a5f6bde1d14faa9e60cde1c8b45c101bc462f34Jens Axboestatic unsigned int nr_thread = 0; 662e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 676eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Listruct io_log *agg_io_log[DDIR_RWDIR_CNT]; 682e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 69a3efc919b0dcc27d65a6e84edca209b91cee173dJens Axboeint groupid = 0; 70a3efc919b0dcc27d65a6e84edca209b91cee173dJens Axboeunsigned int thread_number = 0; 71108fea772db5f1dd91e2fb67737e3e0d36827b76Jens Axboeunsigned int stat_number = 0; 72a3efc919b0dcc27d65a6e84edca209b91cee173dJens Axboeint shm_id = 0; 73a3efc919b0dcc27d65a6e84edca209b91cee173dJens Axboeint temp_stall_ts; 74a3efc919b0dcc27d65a6e84edca209b91cee173dJens Axboeunsigned long done_secs = 0; 7527357187b0899380bae5c6748f2b2d034502e162Jens Axboevolatile int disk_util_exit = 0; 76a3efc919b0dcc27d65a6e84edca209b91cee173dJens Axboe 772e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#define PAGE_ALIGN(buf) \ 78e43606c2b13ad7fc1af2bbe4a61cf8480ee3a532Bruce Cran (char *) (((uintptr_t) (buf) + page_mask) & ~page_mask) 792e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 802e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#define JOB_START_TIMEOUT (5 * 1000) 812e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 822e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic void sig_int(int sig) 832e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 842e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (threads) { 852e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (is_backend) 862e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_server_got_signal(sig); 872e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe else { 882e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_info("\nfio: terminating on signal %d\n", sig); 892e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fflush(stdout); 902e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe exit_value = 128; 912e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 922e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 932e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_terminate_threads(TERMINATE_ALL); 942e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 952e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 962e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 974c6d91e8808fd11f3cee663e42db813b0934c8bbJens Axboestatic void sig_show_status(int sig) 984c6d91e8808fd11f3cee663e42db813b0934c8bbJens Axboe{ 994c6d91e8808fd11f3cee663e42db813b0934c8bbJens Axboe show_running_run_stats(); 1004c6d91e8808fd11f3cee663e42db813b0934c8bbJens Axboe} 1014c6d91e8808fd11f3cee663e42db813b0934c8bbJens Axboe 1022e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic void set_sig_handlers(void) 1032e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 1042e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe struct sigaction act; 1052e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1062e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe memset(&act, 0, sizeof(act)); 1072e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe act.sa_handler = sig_int; 1082e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe act.sa_flags = SA_RESTART; 1092e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe sigaction(SIGINT, &act, NULL); 1102e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1112e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe memset(&act, 0, sizeof(act)); 1122e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe act.sa_handler = sig_int; 1132e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe act.sa_flags = SA_RESTART; 1142e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe sigaction(SIGTERM, &act, NULL); 1152e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1162f694507513e44e59139f57f28e08c5d0772880eBruce Cran/* Windows uses SIGBREAK as a quit signal from other applications */ 1172f694507513e44e59139f57f28e08c5d0772880eBruce Cran#ifdef WIN32 1182f694507513e44e59139f57f28e08c5d0772880eBruce Cran memset(&act, 0, sizeof(act)); 1192f694507513e44e59139f57f28e08c5d0772880eBruce Cran act.sa_handler = sig_int; 1202f694507513e44e59139f57f28e08c5d0772880eBruce Cran act.sa_flags = SA_RESTART; 1212f694507513e44e59139f57f28e08c5d0772880eBruce Cran sigaction(SIGBREAK, &act, NULL); 1222f694507513e44e59139f57f28e08c5d0772880eBruce Cran#endif 1232f694507513e44e59139f57f28e08c5d0772880eBruce Cran 1244c6d91e8808fd11f3cee663e42db813b0934c8bbJens Axboe memset(&act, 0, sizeof(act)); 1254c6d91e8808fd11f3cee663e42db813b0934c8bbJens Axboe act.sa_handler = sig_show_status; 1264c6d91e8808fd11f3cee663e42db813b0934c8bbJens Axboe act.sa_flags = SA_RESTART; 1274c6d91e8808fd11f3cee663e42db813b0934c8bbJens Axboe sigaction(SIGUSR1, &act, NULL); 1284c6d91e8808fd11f3cee663e42db813b0934c8bbJens Axboe 1292e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (is_backend) { 1302e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe memset(&act, 0, sizeof(act)); 1312e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe act.sa_handler = sig_int; 1322e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe act.sa_flags = SA_RESTART; 1332e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe sigaction(SIGPIPE, &act, NULL); 1342e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 1352e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 1362e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1372e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe/* 1382e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * Check if we are above the minimum rate given. 1392e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 1402e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic int __check_min_rate(struct thread_data *td, struct timeval *now, 1412e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe enum fio_ddir ddir) 1422e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 1432e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe unsigned long long bytes = 0; 1442e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe unsigned long iops = 0; 1452e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe unsigned long spent; 1462e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe unsigned long rate; 1472e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe unsigned int ratemin = 0; 1482e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe unsigned int rate_iops = 0; 1492e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe unsigned int rate_iops_min = 0; 1502e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1512e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe assert(ddir_rw(ddir)); 1522e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1532e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!td->o.ratemin[ddir] && !td->o.rate_iops_min[ddir]) 1542e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 0; 1552e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1562e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 1572e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * allow a 2 second settle period in the beginning 1582e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 1592e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (mtime_since(&td->start, now) < 2000) 1602e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 0; 1612e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1622e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe iops += td->this_io_blocks[ddir]; 1632e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe bytes += td->this_io_bytes[ddir]; 1642e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ratemin += td->o.ratemin[ddir]; 1652e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe rate_iops += td->o.rate_iops[ddir]; 1662e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe rate_iops_min += td->o.rate_iops_min[ddir]; 1672e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1682e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 1692e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * if rate blocks is set, sample is running 1702e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 1712e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->rate_bytes[ddir] || td->rate_blocks[ddir]) { 1722e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe spent = mtime_since(&td->lastrate[ddir], now); 1732e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (spent < td->o.ratecycle) 1742e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 0; 1752e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1762e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->o.rate[ddir]) { 1772e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 1782e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * check bandwidth specified rate 1792e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 1802e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (bytes < td->rate_bytes[ddir]) { 1812e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_err("%s: min rate %u not met\n", td->o.name, 1822e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ratemin); 1832e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 1842e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } else { 18549cba9b3e5fa6b30f0fdff52eaf83427ec1baa3fJens Axboe if (spent) 18649cba9b3e5fa6b30f0fdff52eaf83427ec1baa3fJens Axboe rate = ((bytes - td->rate_bytes[ddir]) * 1000) / spent; 18749cba9b3e5fa6b30f0fdff52eaf83427ec1baa3fJens Axboe else 18849cba9b3e5fa6b30f0fdff52eaf83427ec1baa3fJens Axboe rate = 0; 18949cba9b3e5fa6b30f0fdff52eaf83427ec1baa3fJens Axboe 1902e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (rate < ratemin || 1912e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe bytes < td->rate_bytes[ddir]) { 1922e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_err("%s: min rate %u not met, got" 1932e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe " %luKB/sec\n", td->o.name, 1942e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ratemin, rate); 1952e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 1962e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 1972e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 1982e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } else { 1992e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 2002e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * checks iops specified rate 2012e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 2022e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (iops < rate_iops) { 2032e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_err("%s: min iops rate %u not met\n", 2042e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->o.name, rate_iops); 2052e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 2062e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } else { 2074c707a3b3ac73eb5ef00e1664a902fa0a6b69cd0Jens Axboe if (spent) 2084c707a3b3ac73eb5ef00e1664a902fa0a6b69cd0Jens Axboe rate = ((iops - td->rate_blocks[ddir]) * 1000) / spent; 2094c707a3b3ac73eb5ef00e1664a902fa0a6b69cd0Jens Axboe else 2104c707a3b3ac73eb5ef00e1664a902fa0a6b69cd0Jens Axboe rate = 0; 2114c707a3b3ac73eb5ef00e1664a902fa0a6b69cd0Jens Axboe 2122e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (rate < rate_iops_min || 2132e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe iops < td->rate_blocks[ddir]) { 2142e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_err("%s: min iops rate %u not met," 2152e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe " got %lu\n", td->o.name, 2162e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe rate_iops_min, rate); 2172e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 2182e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 2192e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 2202e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 2212e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 2222e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->rate_bytes[ddir] = bytes; 2232e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->rate_blocks[ddir] = iops; 2242e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe memcpy(&td->lastrate[ddir], now, sizeof(*now)); 2252e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 0; 2262e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 2272e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 2282e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic int check_min_rate(struct thread_data *td, struct timeval *now, 229100f49f105126a26de6f1069679c20fd75194188Jens Axboe uint64_t *bytes_done) 2302e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 2312e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int ret = 0; 2322e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 2336eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li if (bytes_done[DDIR_READ]) 2346eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li ret |= __check_min_rate(td, now, DDIR_READ); 2356eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li if (bytes_done[DDIR_WRITE]) 2366eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li ret |= __check_min_rate(td, now, DDIR_WRITE); 2376eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li if (bytes_done[DDIR_TRIM]) 2386eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li ret |= __check_min_rate(td, now, DDIR_TRIM); 2392e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 2402e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return ret; 2412e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 2422e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 2432e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe/* 2442e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * When job exits, we can cancel the in-flight IO if we are using async 2452e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * io. Attempt to do so. 2462e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 2472e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic void cleanup_pending_aio(struct thread_data *td) 2482e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 2492e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int r; 2502e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 2512e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 2522e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * get immediately available events, if any 2532e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 2542e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe r = io_u_queued_complete(td, 0, NULL); 2552e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (r < 0) 2562e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return; 2572e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 2582e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 2592e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * now cancel remaining active events 2602e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 2612e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->io_ops->cancel) { 2622ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe struct io_u *io_u; 2632ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe int i; 2642e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 2652ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe io_u_qiter(&td->io_u_all, io_u, i) { 2662ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe if (io_u->flags & IO_U_F_FLIGHT) { 2672e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe r = td->io_ops->cancel(td, io_u); 2682e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!r) 2692e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe put_io_u(td, io_u); 2702e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 2712e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 2722e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 2732e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 2742e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->cur_depth) 2752e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe r = io_u_queued_complete(td, td->cur_depth, NULL); 2762e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 2772e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 2782e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe/* 2792e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * Helper to handle the final sync of a file. Works just like the normal 2802e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * io path, just does everything sync. 2812e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 2822e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic int fio_io_sync(struct thread_data *td, struct fio_file *f) 2832e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 2842e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe struct io_u *io_u = __get_io_u(td); 2852e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int ret; 2862e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 2872e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!io_u) 2882e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 2892e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 2902e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe io_u->ddir = DDIR_SYNC; 2912e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe io_u->file = f; 2922e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 2932e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td_io_prep(td, io_u)) { 2942e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe put_io_u(td, io_u); 2952e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 2962e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 2972e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 2982e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboerequeue: 2992e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ret = td_io_queue(td, io_u); 3002e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (ret < 0) { 3012e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_verror(td, io_u->error, "td_io_queue"); 3022e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe put_io_u(td, io_u); 3032e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 3042e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } else if (ret == FIO_Q_QUEUED) { 3052e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (io_u_queued_complete(td, 1, NULL) < 0) 3062e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 3072e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } else if (ret == FIO_Q_COMPLETED) { 3082e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (io_u->error) { 3092e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_verror(td, io_u->error, "td_io_queue"); 3102e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 3112e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 3122e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 3132e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (io_u_sync_complete(td, io_u, NULL) < 0) 3142e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 3152e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } else if (ret == FIO_Q_BUSY) { 3162e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td_io_commit(td)) 3172e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 3182e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe goto requeue; 3192e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 3202e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 3212e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 0; 3222e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 323a3efc919b0dcc27d65a6e84edca209b91cee173dJens Axboe 32461ee0f86fc428db9403f0c1123e4c2f125152466Jens Axboestatic int fio_file_fsync(struct thread_data *td, struct fio_file *f) 32561ee0f86fc428db9403f0c1123e4c2f125152466Jens Axboe{ 32661ee0f86fc428db9403f0c1123e4c2f125152466Jens Axboe int ret; 32761ee0f86fc428db9403f0c1123e4c2f125152466Jens Axboe 32861ee0f86fc428db9403f0c1123e4c2f125152466Jens Axboe if (fio_file_open(f)) 32961ee0f86fc428db9403f0c1123e4c2f125152466Jens Axboe return fio_io_sync(td, f); 33061ee0f86fc428db9403f0c1123e4c2f125152466Jens Axboe 33161ee0f86fc428db9403f0c1123e4c2f125152466Jens Axboe if (td_io_open_file(td, f)) 33261ee0f86fc428db9403f0c1123e4c2f125152466Jens Axboe return 1; 33361ee0f86fc428db9403f0c1123e4c2f125152466Jens Axboe 33461ee0f86fc428db9403f0c1123e4c2f125152466Jens Axboe ret = fio_io_sync(td, f); 33561ee0f86fc428db9403f0c1123e4c2f125152466Jens Axboe td_io_close_file(td, f); 33661ee0f86fc428db9403f0c1123e4c2f125152466Jens Axboe return ret; 33761ee0f86fc428db9403f0c1123e4c2f125152466Jens Axboe} 33861ee0f86fc428db9403f0c1123e4c2f125152466Jens Axboe 3392e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic inline void __update_tv_cache(struct thread_data *td) 3402e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 3412e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_gettime(&td->tv_cache, NULL); 3422e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 3432e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 3442e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic inline void update_tv_cache(struct thread_data *td) 3452e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 3462e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if ((++td->tv_cache_nr & td->tv_cache_mask) == td->tv_cache_mask) 3472e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe __update_tv_cache(td); 3482e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 3492e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 3502e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic inline int runtime_exceeded(struct thread_data *td, struct timeval *t) 3512e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 3522e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (in_ramp_time(td)) 3532e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 0; 3542e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!td->o.timeout) 3552e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 0; 3560de5b26f6e177aacac0683306c47e0cbaf58b0b6Jens Axboe if (utime_since(&td->epoch, t) >= td->o.timeout) 3572e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 3582e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 3592e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 0; 3602e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 3612e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 3622e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic int break_on_this_error(struct thread_data *td, enum fio_ddir ddir, 3632e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int *retptr) 3642e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 3652e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int ret = *retptr; 3662e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 3672e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (ret < 0 || td->error) { 3688b28bd41375930664a0ff9ff9b101a88ac416ac5Dmitry Monakhov int err = td->error; 3698b28bd41375930664a0ff9ff9b101a88ac416ac5Dmitry Monakhov enum error_type_bit eb; 3702e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 3712e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (ret < 0) 3722e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe err = -ret; 3732e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 3748b28bd41375930664a0ff9ff9b101a88ac416ac5Dmitry Monakhov eb = td_error_type(ddir, err); 3758b28bd41375930664a0ff9ff9b101a88ac416ac5Dmitry Monakhov if (!(td->o.continue_on_error & (1 << eb))) 3762e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 3772e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 3788b28bd41375930664a0ff9ff9b101a88ac416ac5Dmitry Monakhov if (td_non_fatal_error(td, eb, err)) { 3792e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 3802e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * Continue with the I/Os in case of 3812e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * a non fatal error. 3822e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 3832e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe update_error_count(td, err); 3842e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_clear_error(td); 3852e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe *retptr = 0; 3862e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 0; 3872e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } else if (td->o.fill_device && err == ENOSPC) { 3882e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 3892e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * We expect to hit this error if 3902e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * fill_device option is set. 3912e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 3922e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_clear_error(td); 3932e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->terminate = 1; 3942e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 3952e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } else { 3962e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 3972e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * Stop the I/O in case of a fatal 3982e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * error. 3992e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 4002e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe update_error_count(td, err); 4012e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 4022e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 4032e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 4042e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 4052e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 0; 4062e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 4072e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 408c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboestatic void check_update_rusage(struct thread_data *td) 409c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe{ 410c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe if (td->update_rusage) { 411c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe td->update_rusage = 0; 412c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe update_rusage_stat(td); 413c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe fio_mutex_up(td->rusage_sem); 414c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe } 415c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe} 416c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe 4172e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe/* 4182e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * The main verify engine. Runs over the writes we previously submitted, 4192e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * reads the blocks back in, and checks the crc/md5 of the data. 4202e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 421100f49f105126a26de6f1069679c20fd75194188Jens Axboestatic void do_verify(struct thread_data *td, uint64_t verify_bytes) 4222e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 423100f49f105126a26de6f1069679c20fd75194188Jens Axboe uint64_t bytes_done[DDIR_RWDIR_CNT] = { 0, 0, 0 }; 4242e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe struct fio_file *f; 4252e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe struct io_u *io_u; 4262e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int ret, min_events; 4272e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe unsigned int i; 4282e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 4292e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe dprint(FD_VERIFY, "starting loop\n"); 4302e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 4312e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 4322e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * sync io first and invalidate cache, to make sure we really 4332e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * read from disk. 4342e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 4352e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe for_each_file(td, f, i) { 4362e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!fio_file_open(f)) 4372e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe continue; 4382e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (fio_io_sync(td, f)) 4392e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 4402e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (file_invalidate_cache(td, f)) 4412e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 4422e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 4432e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 444c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe check_update_rusage(td); 445c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe 4462e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->error) 4472e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return; 4482e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 4492e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_set_runstate(td, TD_VERIFYING); 4502e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 4512e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe io_u = NULL; 4522e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe while (!td->terminate) { 453fbccf46c939c9fee9f738da31c2e22791be5eab7Jens Axboe enum fio_ddir ddir; 4542e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int ret2, full; 4552e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 4562e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe update_tv_cache(td); 457c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe check_update_rusage(td); 4582e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 4592e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (runtime_exceeded(td, &td->tv_cache)) { 4602e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe __update_tv_cache(td); 4612e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (runtime_exceeded(td, &td->tv_cache)) { 4622e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->terminate = 1; 4632e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 4642e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 4652e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 4662e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 4679e684a4976b7934f5ce011ea281dfef3352e5738Dan Ehrenberg if (flow_threshold_exceeded(td)) 4689e684a4976b7934f5ce011ea281dfef3352e5738Dan Ehrenberg continue; 4699e684a4976b7934f5ce011ea281dfef3352e5738Dan Ehrenberg 47044cbc6daaae3674ea5d0a113b66596ca24f372e0Jens Axboe if (!td->o.experimental_verify) { 47144cbc6daaae3674ea5d0a113b66596ca24f372e0Jens Axboe io_u = __get_io_u(td); 47244cbc6daaae3674ea5d0a113b66596ca24f372e0Jens Axboe if (!io_u) 47344cbc6daaae3674ea5d0a113b66596ca24f372e0Jens Axboe break; 4742e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 47544cbc6daaae3674ea5d0a113b66596ca24f372e0Jens Axboe if (get_next_verify(td, io_u)) { 47644cbc6daaae3674ea5d0a113b66596ca24f372e0Jens Axboe put_io_u(td, io_u); 47744cbc6daaae3674ea5d0a113b66596ca24f372e0Jens Axboe break; 47844cbc6daaae3674ea5d0a113b66596ca24f372e0Jens Axboe } 4792e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 48044cbc6daaae3674ea5d0a113b66596ca24f372e0Jens Axboe if (td_io_prep(td, io_u)) { 48144cbc6daaae3674ea5d0a113b66596ca24f372e0Jens Axboe put_io_u(td, io_u); 48244cbc6daaae3674ea5d0a113b66596ca24f372e0Jens Axboe break; 48344cbc6daaae3674ea5d0a113b66596ca24f372e0Jens Axboe } 48444cbc6daaae3674ea5d0a113b66596ca24f372e0Jens Axboe } else { 485100f49f105126a26de6f1069679c20fd75194188Jens Axboe if (ddir_rw_sum(bytes_done) + td->o.rw_min_bs > verify_bytes) 486100f49f105126a26de6f1069679c20fd75194188Jens Axboe break; 487100f49f105126a26de6f1069679c20fd75194188Jens Axboe 488bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe while ((io_u = get_io_u(td)) != NULL) { 489002fe73409d1e3d5e7dfe2885f75885bfaf506bcJens Axboe if (IS_ERR(io_u)) { 490002fe73409d1e3d5e7dfe2885f75885bfaf506bcJens Axboe io_u = NULL; 491002fe73409d1e3d5e7dfe2885f75885bfaf506bcJens Axboe ret = FIO_Q_BUSY; 492002fe73409d1e3d5e7dfe2885f75885bfaf506bcJens Axboe goto reap; 493002fe73409d1e3d5e7dfe2885f75885bfaf506bcJens Axboe } 494002fe73409d1e3d5e7dfe2885f75885bfaf506bcJens Axboe 495bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe /* 496bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe * We are only interested in the places where 497bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe * we wrote or trimmed IOs. Turn those into 498bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe * reads for verification purposes. 499bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe */ 500bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe if (io_u->ddir == DDIR_READ) { 501bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe /* 502bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe * Pretend we issued it for rwmix 503bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe * accounting 504bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe */ 505bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe td->io_issues[DDIR_READ]++; 506bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe put_io_u(td, io_u); 507bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe continue; 508bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe } else if (io_u->ddir == DDIR_TRIM) { 509bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe io_u->ddir = DDIR_READ; 510bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe io_u->flags |= IO_U_F_TRIMMED; 511bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe break; 512bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe } else if (io_u->ddir == DDIR_WRITE) { 513bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe io_u->ddir = DDIR_READ; 514bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe break; 515bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe } else { 516bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe put_io_u(td, io_u); 517bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe continue; 518bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe } 519bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe } 52044cbc6daaae3674ea5d0a113b66596ca24f372e0Jens Axboe 521bcd5abfa9f230bbe4365dad1289fdea1f5509f74Jens Axboe if (!io_u) 52244cbc6daaae3674ea5d0a113b66596ca24f372e0Jens Axboe break; 5232e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 5242e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 5252e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->o.verify_async) 5262e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe io_u->end_io = verify_io_u_async; 5272e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe else 5282e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe io_u->end_io = verify_io_u; 5292e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 530fbccf46c939c9fee9f738da31c2e22791be5eab7Jens Axboe ddir = io_u->ddir; 531fbccf46c939c9fee9f738da31c2e22791be5eab7Jens Axboe 5322e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ret = td_io_queue(td, io_u); 5332e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe switch (ret) { 5342e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe case FIO_Q_COMPLETED: 5352e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (io_u->error) { 5362e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ret = -io_u->error; 5372e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe clear_io_u(td, io_u); 5382e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } else if (io_u->resid) { 5392e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int bytes = io_u->xfer_buflen - io_u->resid; 5402e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 5412e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 5422e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * zero read, fail 5432e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 5442e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!bytes) { 5452e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_verror(td, EIO, "full resid"); 5462e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe put_io_u(td, io_u); 5472e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 5482e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 5492e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 5502e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe io_u->xfer_buflen = io_u->resid; 5512e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe io_u->xfer_buf += bytes; 5522e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe io_u->offset += bytes; 5532e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 5542e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (ddir_rw(io_u->ddir)) 5552e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->ts.short_io_u[io_u->ddir]++; 5562e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 5572e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe f = io_u->file; 5582e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (io_u->offset == f->real_file_size) 5592e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe goto sync_done; 5602e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 5612e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe requeue_io_u(td, &io_u); 5622e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } else { 5632e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboesync_done: 564100f49f105126a26de6f1069679c20fd75194188Jens Axboe ret = io_u_sync_complete(td, io_u, bytes_done); 5652e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (ret < 0) 5662e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 5672e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 5682e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe continue; 5692e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe case FIO_Q_QUEUED: 5702e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 5712e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe case FIO_Q_BUSY: 5722e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe requeue_io_u(td, &io_u); 5732e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ret2 = td_io_commit(td); 5742e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (ret2 < 0) 5752e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ret = ret2; 5762e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 5772e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe default: 5782e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe assert(ret < 0); 5792e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_verror(td, -ret, "td_io_queue"); 5802e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 5812e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 5822e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 583fbccf46c939c9fee9f738da31c2e22791be5eab7Jens Axboe if (break_on_this_error(td, ddir, &ret)) 5842e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 5852e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 5862e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 5872e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * if we can queue more, do so. but check if there are 5882e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * completed io_u's first. Note that we can get BUSY even 5892e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * without IO queued, if the system is resource starved. 5902e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 591002fe73409d1e3d5e7dfe2885f75885bfaf506bcJens Axboereap: 5922e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe full = queue_full(td) || (ret == FIO_Q_BUSY && td->cur_depth); 5932e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (full || !td->o.iodepth_batch_complete) { 5942e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe min_events = min(td->o.iodepth_batch_complete, 5952e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->cur_depth); 5968a74b56d7aef44ea75d7abf910ad75378b438fe3Jens Axboe /* 5978a74b56d7aef44ea75d7abf910ad75378b438fe3Jens Axboe * if the queue is full, we MUST reap at least 1 event 5988a74b56d7aef44ea75d7abf910ad75378b438fe3Jens Axboe */ 5998a74b56d7aef44ea75d7abf910ad75378b438fe3Jens Axboe if (full && !min_events) 6002e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe min_events = 1; 6012e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 6022e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe do { 6032e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 6042e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * Reap required number of io units, if any, 6052e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * and do the verification on them through 6062e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * the callback handler 6072e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 608100f49f105126a26de6f1069679c20fd75194188Jens Axboe if (io_u_queued_complete(td, min_events, bytes_done) < 0) { 6092e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ret = -1; 6102e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 6112e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 6122e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } while (full && (td->cur_depth > td->o.iodepth_low)); 6132e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 6142e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (ret < 0) 6152e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 6162e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 6172e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 618c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe check_update_rusage(td); 619c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe 6202e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!td->error) { 6212e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe min_events = td->cur_depth; 6222e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 6232e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (min_events) 6242e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ret = io_u_queued_complete(td, min_events, NULL); 6252e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } else 6262e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe cleanup_pending_aio(td); 6272e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 6282e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_set_runstate(td, TD_RUNNING); 6292e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 6302e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe dprint(FD_VERIFY, "exiting loop\n"); 6312e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 6322e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 6333939fe853fd146378811a05b78731d7f20d0152fJens Axboestatic unsigned int exceeds_number_ios(struct thread_data *td) 6343939fe853fd146378811a05b78731d7f20d0152fJens Axboe{ 6353939fe853fd146378811a05b78731d7f20d0152fJens Axboe unsigned long long number_ios; 6363939fe853fd146378811a05b78731d7f20d0152fJens Axboe 6373939fe853fd146378811a05b78731d7f20d0152fJens Axboe if (!td->o.number_ios) 6383939fe853fd146378811a05b78731d7f20d0152fJens Axboe return 0; 6393939fe853fd146378811a05b78731d7f20d0152fJens Axboe 6403939fe853fd146378811a05b78731d7f20d0152fJens Axboe number_ios = ddir_rw_sum(td->this_io_blocks); 6413939fe853fd146378811a05b78731d7f20d0152fJens Axboe number_ios += td->io_u_queued + td->io_u_in_flight; 6423939fe853fd146378811a05b78731d7f20d0152fJens Axboe 6433939fe853fd146378811a05b78731d7f20d0152fJens Axboe return number_ios >= td->o.number_ios; 6443939fe853fd146378811a05b78731d7f20d0152fJens Axboe} 6453939fe853fd146378811a05b78731d7f20d0152fJens Axboe 646f7078f7b517ace62bdf22967c18525b5b6c68b85Jens Axboestatic int io_bytes_exceeded(struct thread_data *td) 647f7078f7b517ace62bdf22967c18525b5b6c68b85Jens Axboe{ 64877731b29c35286a43ca6109ebf13389169ec20e9Jens Axboe unsigned long long bytes, limit; 649f7078f7b517ace62bdf22967c18525b5b6c68b85Jens Axboe 650f7078f7b517ace62bdf22967c18525b5b6c68b85Jens Axboe if (td_rw(td)) 6516eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li bytes = td->this_io_bytes[DDIR_READ] + td->this_io_bytes[DDIR_WRITE]; 652f7078f7b517ace62bdf22967c18525b5b6c68b85Jens Axboe else if (td_write(td)) 6536eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li bytes = td->this_io_bytes[DDIR_WRITE]; 6546eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li else if (td_read(td)) 6556eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li bytes = td->this_io_bytes[DDIR_READ]; 656f7078f7b517ace62bdf22967c18525b5b6c68b85Jens Axboe else 6576eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li bytes = td->this_io_bytes[DDIR_TRIM]; 658f7078f7b517ace62bdf22967c18525b5b6c68b85Jens Axboe 65977731b29c35286a43ca6109ebf13389169ec20e9Jens Axboe if (td->o.io_limit) 66077731b29c35286a43ca6109ebf13389169ec20e9Jens Axboe limit = td->o.io_limit; 66177731b29c35286a43ca6109ebf13389169ec20e9Jens Axboe else 66277731b29c35286a43ca6109ebf13389169ec20e9Jens Axboe limit = td->o.size; 66377731b29c35286a43ca6109ebf13389169ec20e9Jens Axboe 66477731b29c35286a43ca6109ebf13389169ec20e9Jens Axboe return bytes >= limit || exceeds_number_ios(td); 665f7078f7b517ace62bdf22967c18525b5b6c68b85Jens Axboe} 666f7078f7b517ace62bdf22967c18525b5b6c68b85Jens Axboe 6672e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe/* 6682e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * Main IO worker function. It retrieves io_u's to process and queues 6692e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * and reaps them, checking for rate and errors along the way. 670100f49f105126a26de6f1069679c20fd75194188Jens Axboe * 671100f49f105126a26de6f1069679c20fd75194188Jens Axboe * Returns number of bytes written and trimmed. 6722e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 673100f49f105126a26de6f1069679c20fd75194188Jens Axboestatic uint64_t do_io(struct thread_data *td) 6742e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 675100f49f105126a26de6f1069679c20fd75194188Jens Axboe uint64_t bytes_done[DDIR_RWDIR_CNT] = { 0, 0, 0 }; 6762e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe unsigned int i; 6772e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int ret = 0; 678c2703bf35e146872d12f85c9e6490008b5fabb34Jens Axboe uint64_t total_bytes, bytes_issued = 0; 6792e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 6802e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (in_ramp_time(td)) 6812e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_set_runstate(td, TD_RAMP); 6822e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe else 6832e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_set_runstate(td, TD_RUNNING); 6842e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 6853e260a46ea9a8de224c3d0a29a608da3440f284aJens Axboe lat_target_init(td); 6863e260a46ea9a8de224c3d0a29a608da3440f284aJens Axboe 68778a6469cfc9094763320e61c60f9aaef0ece9689Jens Axboe /* 68878a6469cfc9094763320e61c60f9aaef0ece9689Jens Axboe * If verify_backlog is enabled, we'll run the verify in this 68978a6469cfc9094763320e61c60f9aaef0ece9689Jens Axboe * handler as well. For that case, we may need up to twice the 69078a6469cfc9094763320e61c60f9aaef0ece9689Jens Axboe * amount of bytes. 69178a6469cfc9094763320e61c60f9aaef0ece9689Jens Axboe */ 692c2703bf35e146872d12f85c9e6490008b5fabb34Jens Axboe total_bytes = td->o.size; 69378a6469cfc9094763320e61c60f9aaef0ece9689Jens Axboe if (td->o.verify != VERIFY_NONE && 69478a6469cfc9094763320e61c60f9aaef0ece9689Jens Axboe (td_write(td) && td->o.verify_backlog)) 695c2703bf35e146872d12f85c9e6490008b5fabb34Jens Axboe total_bytes += td->o.size; 696c2703bf35e146872d12f85c9e6490008b5fabb34Jens Axboe 697f7078f7b517ace62bdf22967c18525b5b6c68b85Jens Axboe while ((td->o.read_iolog_file && !flist_empty(&td->io_log_list)) || 698c04e4661e4da3b6079f415897e4507cf8e610c54Daniel Ehrenberg (!flist_empty(&td->trim_list)) || !io_bytes_exceeded(td) || 699c04e4661e4da3b6079f415897e4507cf8e610c54Daniel Ehrenberg td->o.time_based) { 7002e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe struct timeval comp_time; 7012e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int min_evts = 0; 7022e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe struct io_u *io_u; 7032e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int ret2, full; 7042e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe enum fio_ddir ddir; 7052e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 706c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe check_update_rusage(td); 707c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe 7087d7803fa331bec4b53edf7bfa3c70ccd7f2920d9Yufei Ren if (td->terminate || td->done) 7092e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 7102e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 7112e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe update_tv_cache(td); 7122e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 7132e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (runtime_exceeded(td, &td->tv_cache)) { 7142e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe __update_tv_cache(td); 7152e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (runtime_exceeded(td, &td->tv_cache)) { 7162e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->terminate = 1; 7172e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 7182e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 7192e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 7202e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 7219e684a4976b7934f5ce011ea281dfef3352e5738Dan Ehrenberg if (flow_threshold_exceeded(td)) 7229e684a4976b7934f5ce011ea281dfef3352e5738Dan Ehrenberg continue; 7239e684a4976b7934f5ce011ea281dfef3352e5738Dan Ehrenberg 724c2703bf35e146872d12f85c9e6490008b5fabb34Jens Axboe if (bytes_issued >= total_bytes) 72520876c53b5d32f2da9049af5e7fb102133946981Juan Casse break; 72620876c53b5d32f2da9049af5e7fb102133946981Juan Casse 7272e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe io_u = get_io_u(td); 728002fe73409d1e3d5e7dfe2885f75885bfaf506bcJens Axboe if (IS_ERR_OR_NULL(io_u)) { 729002fe73409d1e3d5e7dfe2885f75885bfaf506bcJens Axboe int err = PTR_ERR(io_u); 730002fe73409d1e3d5e7dfe2885f75885bfaf506bcJens Axboe 731002fe73409d1e3d5e7dfe2885f75885bfaf506bcJens Axboe io_u = NULL; 732002fe73409d1e3d5e7dfe2885f75885bfaf506bcJens Axboe if (err == -EBUSY) { 733002fe73409d1e3d5e7dfe2885f75885bfaf506bcJens Axboe ret = FIO_Q_BUSY; 734002fe73409d1e3d5e7dfe2885f75885bfaf506bcJens Axboe goto reap; 735002fe73409d1e3d5e7dfe2885f75885bfaf506bcJens Axboe } 7363e260a46ea9a8de224c3d0a29a608da3440f284aJens Axboe if (td->o.latency_target) 7373e260a46ea9a8de224c3d0a29a608da3440f284aJens Axboe goto reap; 7382e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 7393e260a46ea9a8de224c3d0a29a608da3440f284aJens Axboe } 7402e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 7412e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ddir = io_u->ddir; 7422e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 7432e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 74482af2a7ca1a543b41c003de69d5e3c36860f47d5Jens Axboe * Add verification end_io handler if: 74582af2a7ca1a543b41c003de69d5e3c36860f47d5Jens Axboe * - Asked to verify (!td_rw(td)) 74682af2a7ca1a543b41c003de69d5e3c36860f47d5Jens Axboe * - Or the io_u is from our verify list (mixed write/ver) 7472e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 7482e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->o.verify != VERIFY_NONE && io_u->ddir == DDIR_READ && 74982af2a7ca1a543b41c003de69d5e3c36860f47d5Jens Axboe ((io_u->flags & IO_U_F_VER_LIST) || !td_rw(td))) { 750c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat 751c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat if (!td->o.verify_pattern_bytes) { 752c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat io_u->rand_seed = __rand(&td->__verify_state); 753c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat if (sizeof(int) != sizeof(long *)) 754c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat io_u->rand_seed *= __rand(&td->__verify_state); 755c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat } 756c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat 7572e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->o.verify_async) 7582e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe io_u->end_io = verify_io_u_async; 7592e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe else 7602e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe io_u->end_io = verify_io_u; 7612e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_set_runstate(td, TD_VERIFYING); 7622e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } else if (in_ramp_time(td)) 7632e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_set_runstate(td, TD_RAMP); 7642e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe else 7652e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_set_runstate(td, TD_RUNNING); 7662e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 7679a50c5c5650fdd9f3117e3745f16cc63c5b267b7Puthikorn Voravootivat /* 768f940128526dbe468a1951cce10c2fe5dbd23875fJens Axboe * Always log IO before it's issued, so we know the specific 769f940128526dbe468a1951cce10c2fe5dbd23875fJens Axboe * order of it. The logged unit will track when the IO has 770f940128526dbe468a1951cce10c2fe5dbd23875fJens Axboe * completed. 7719a50c5c5650fdd9f3117e3745f16cc63c5b267b7Puthikorn Voravootivat */ 772c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat if (td_write(td) && io_u->ddir == DDIR_WRITE && 773c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat td->o.do_verify && 774c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat td->o.verify != VERIFY_NONE && 775f940128526dbe468a1951cce10c2fe5dbd23875fJens Axboe !td->o.experimental_verify) 776c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat log_io_piece(td, io_u); 777c4b6117b236370b174f3e3d2e299bf8b3733be97Puthikorn Voravootivat 7782e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ret = td_io_queue(td, io_u); 7792e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe switch (ret) { 7802e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe case FIO_Q_COMPLETED: 7812e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (io_u->error) { 7822e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ret = -io_u->error; 783890b665611751661aac1030603175bb7bbde34bcJens Axboe unlog_io_piece(td, io_u); 7842e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe clear_io_u(td, io_u); 7852e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } else if (io_u->resid) { 7862e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int bytes = io_u->xfer_buflen - io_u->resid; 7872e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe struct fio_file *f = io_u->file; 7882e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 78920876c53b5d32f2da9049af5e7fb102133946981Juan Casse bytes_issued += bytes; 790890b665611751661aac1030603175bb7bbde34bcJens Axboe 791890b665611751661aac1030603175bb7bbde34bcJens Axboe trim_io_piece(td, io_u); 792890b665611751661aac1030603175bb7bbde34bcJens Axboe 7932e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 7942e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * zero read, fail 7952e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 7962e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!bytes) { 797890b665611751661aac1030603175bb7bbde34bcJens Axboe unlog_io_piece(td, io_u); 7982e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_verror(td, EIO, "full resid"); 7992e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe put_io_u(td, io_u); 8002e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 8012e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 8022e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 8032e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe io_u->xfer_buflen = io_u->resid; 8042e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe io_u->xfer_buf += bytes; 8052e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe io_u->offset += bytes; 8062e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 8072e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (ddir_rw(io_u->ddir)) 8082e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->ts.short_io_u[io_u->ddir]++; 8092e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 8102e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (io_u->offset == f->real_file_size) 8112e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe goto sync_done; 8122e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 8132e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe requeue_io_u(td, &io_u); 8142e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } else { 8152e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboesync_done: 8166eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li if (__should_check_rate(td, DDIR_READ) || 8176eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li __should_check_rate(td, DDIR_WRITE) || 8186eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li __should_check_rate(td, DDIR_TRIM)) 8192e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_gettime(&comp_time, NULL); 8202e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 8212e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ret = io_u_sync_complete(td, io_u, bytes_done); 8222e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (ret < 0) 8232e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 82420876c53b5d32f2da9049af5e7fb102133946981Juan Casse bytes_issued += io_u->xfer_buflen; 8252e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 8262e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 8272e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe case FIO_Q_QUEUED: 8282e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 8292e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * if the engine doesn't have a commit hook, 8302e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * the io_u is really queued. if it does have such 8312e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * a hook, it has to call io_u_queued() itself. 8322e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 8332e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->io_ops->commit == NULL) 8342e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe io_u_queued(td, io_u); 83520876c53b5d32f2da9049af5e7fb102133946981Juan Casse bytes_issued += io_u->xfer_buflen; 8362e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 8372e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe case FIO_Q_BUSY: 838890b665611751661aac1030603175bb7bbde34bcJens Axboe unlog_io_piece(td, io_u); 8392e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe requeue_io_u(td, &io_u); 8402e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ret2 = td_io_commit(td); 8412e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (ret2 < 0) 8422e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ret = ret2; 8432e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 8442e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe default: 8452e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe assert(ret < 0); 8462e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe put_io_u(td, io_u); 8472e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 8482e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 8492e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 8502e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (break_on_this_error(td, ddir, &ret)) 8512e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 8522e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 8532e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 8542e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * See if we need to complete some commands. Note that we 8552e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * can get BUSY even without IO queued, if the system is 8562e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * resource starved. 8572e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 8583e260a46ea9a8de224c3d0a29a608da3440f284aJens Axboereap: 8592e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe full = queue_full(td) || (ret == FIO_Q_BUSY && td->cur_depth); 8602e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (full || !td->o.iodepth_batch_complete) { 8612e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe min_evts = min(td->o.iodepth_batch_complete, 8622e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->cur_depth); 8638a74b56d7aef44ea75d7abf910ad75378b438fe3Jens Axboe /* 8648a74b56d7aef44ea75d7abf910ad75378b438fe3Jens Axboe * if the queue is full, we MUST reap at least 1 event 8658a74b56d7aef44ea75d7abf910ad75378b438fe3Jens Axboe */ 8668a74b56d7aef44ea75d7abf910ad75378b438fe3Jens Axboe if (full && !min_evts) 8672e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe min_evts = 1; 8682e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 8696eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li if (__should_check_rate(td, DDIR_READ) || 8706eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li __should_check_rate(td, DDIR_WRITE) || 8716eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li __should_check_rate(td, DDIR_TRIM)) 8722e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_gettime(&comp_time, NULL); 8732e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 8742e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe do { 8752e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ret = io_u_queued_complete(td, min_evts, bytes_done); 8762e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (ret < 0) 8772e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 8782e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 8792e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } while (full && (td->cur_depth > td->o.iodepth_low)); 8802e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 8812e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 8822e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (ret < 0) 8832e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 884d5abee06d57595679bf8f5fbe70c532a3d8cfdbaJens Axboe if (!ddir_rw_sum(bytes_done) && !(td->io_ops->flags & FIO_NOIO)) 8852e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe continue; 8862e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 8872e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!in_ramp_time(td) && should_check_rate(td, bytes_done)) { 8882e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (check_min_rate(td, &comp_time, bytes_done)) { 8892e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (exitall_on_terminate) 8902e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_terminate_threads(td->groupid); 8912e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_verror(td, EIO, "check_min_rate"); 8922e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 8932e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 8942e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 8953e260a46ea9a8de224c3d0a29a608da3440f284aJens Axboe if (!in_ramp_time(td) && td->o.latency_target) 8963e260a46ea9a8de224c3d0a29a608da3440f284aJens Axboe lat_target_check(td); 8972e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 8982e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->o.thinktime) { 8992e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe unsigned long long b; 9002e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 901342f4be43b2e74cf2f9a2102afdad11c8951b73fJens Axboe b = ddir_rw_sum(td->io_blocks); 9022e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!(b % td->o.thinktime_blocks)) { 9032e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int left; 9042e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 905002e7183cb86d6100efef690b6fa90bf0988b084Jens Axboe io_u_quiesce(td); 906002e7183cb86d6100efef690b6fa90bf0988b084Jens Axboe 9072e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->o.thinktime_spin) 9082e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe usec_spin(td->o.thinktime_spin); 9092e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 9102e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe left = td->o.thinktime - td->o.thinktime_spin; 9112e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (left) 9122e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe usec_sleep(td, left); 9132e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 9142e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 9152e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 9162e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 917c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe check_update_rusage(td); 918c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe 9192e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->trim_entries) 9204e0a8fa2593006505b7f4e18931a201d221b49e9Jens Axboe log_err("fio: %lu trim entries leaked?\n", td->trim_entries); 9212e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 9222e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->o.fill_device && td->error == ENOSPC) { 9232e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->error = 0; 9242e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->terminate = 1; 9252e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 9262e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!td->error) { 9272e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe struct fio_file *f; 9282e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 9292e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe i = td->cur_depth; 9302e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (i) { 9315bd5f71a3e54950fb4bee64c7ad5edc67e40b8c8Jens Axboe ret = io_u_queued_complete(td, i, bytes_done); 9322e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->o.fill_device && td->error == ENOSPC) 9332e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->error = 0; 9342e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 9352e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 9362e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (should_fsync(td) && td->o.end_fsync) { 9372e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_set_runstate(td, TD_FSYNCING); 9382e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 9392e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe for_each_file(td, f, i) { 94061ee0f86fc428db9403f0c1123e4c2f125152466Jens Axboe if (!fio_file_fsync(td, f)) 9412e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe continue; 94261ee0f86fc428db9403f0c1123e4c2f125152466Jens Axboe 94361ee0f86fc428db9403f0c1123e4c2f125152466Jens Axboe log_err("fio: end_fsync failed for file %s\n", 94461ee0f86fc428db9403f0c1123e4c2f125152466Jens Axboe f->file_name); 9452e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 9462e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 9472e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } else 9482e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe cleanup_pending_aio(td); 9492e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 9502e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 9512e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * stop job if we failed doing any IO 9522e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 953342f4be43b2e74cf2f9a2102afdad11c8951b73fJens Axboe if (!ddir_rw_sum(td->this_io_bytes)) 9542e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->done = 1; 955100f49f105126a26de6f1069679c20fd75194188Jens Axboe 956100f49f105126a26de6f1069679c20fd75194188Jens Axboe return bytes_done[DDIR_WRITE] + bytes_done[DDIR_TRIM]; 9572e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 9582e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 9592e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic void cleanup_io_u(struct thread_data *td) 9602e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 9612e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe struct io_u *io_u; 9622e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 9632ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe while ((io_u = io_u_qpop(&td->io_u_freelist)) != NULL) { 964c73ed24623ad94a5254e8302298797aba2eb1e9aJens Axboe 965c73ed24623ad94a5254e8302298797aba2eb1e9aJens Axboe if (td->io_ops->io_u_free) 966c73ed24623ad94a5254e8302298797aba2eb1e9aJens Axboe td->io_ops->io_u_free(td, io_u); 967c73ed24623ad94a5254e8302298797aba2eb1e9aJens Axboe 9682e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_memfree(io_u, sizeof(*io_u)); 9692e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 9702e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 9712e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe free_io_mem(td); 9722ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe 9732ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe io_u_rexit(&td->io_u_requeues); 9742ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe io_u_qexit(&td->io_u_freelist); 9752ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe io_u_qexit(&td->io_u_all); 9762e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 9772e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 9782e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic int init_io_u(struct thread_data *td) 9792e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 9802e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe struct io_u *io_u; 9819c42684e32325da26e862280388798343c5f1305Jens Axboe unsigned int max_bs, min_write; 9822e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int cl_align, i, max_units; 9832ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe int data_xfer = 1, err; 9842e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe char *p; 9852e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 9862e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe max_units = td->o.iodepth; 98774f4b020b3c8053ff319c3da900a5cf07e51c8ebJens Axboe max_bs = td_max_bs(td); 9889c42684e32325da26e862280388798343c5f1305Jens Axboe min_write = td->o.min_bs[DDIR_WRITE]; 9892e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->orig_buffer_size = (unsigned long long) max_bs 9902e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * (unsigned long long) max_units; 9912e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 99288045e04fdd81dbb584d70f9faeb1f6498a85159Dmitry Monakhov if ((td->io_ops->flags & FIO_NOIO) || !(td_read(td) || td_write(td))) 99359d8d0f5fb939ef23e4aa495dbe8fabd59d5bf25Jens Axboe data_xfer = 0; 99459d8d0f5fb939ef23e4aa495dbe8fabd59d5bf25Jens Axboe 9952ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe err = 0; 9962ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe err += io_u_rinit(&td->io_u_requeues, td->o.iodepth); 9972ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe err += io_u_qinit(&td->io_u_freelist, td->o.iodepth); 9982ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe err += io_u_qinit(&td->io_u_all, td->o.iodepth); 9992ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe 10002ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe if (err) { 10012ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe log_err("fio: failed setting up IO queues\n"); 10022ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe return 1; 10032ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe } 10042ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe 1005fd8a09b8f4a7e0da67fe64cfb1d023bfb500d7ebpeter chang /* 1006fd8a09b8f4a7e0da67fe64cfb1d023bfb500d7ebpeter chang * if we may later need to do address alignment, then add any 1007fd8a09b8f4a7e0da67fe64cfb1d023bfb500d7ebpeter chang * possible adjustment here so that we don't cause a buffer 1008fd8a09b8f4a7e0da67fe64cfb1d023bfb500d7ebpeter chang * overflow later. this adjustment may be too much if we get 1009fd8a09b8f4a7e0da67fe64cfb1d023bfb500d7ebpeter chang * lucky and the allocator gives us an aligned address. 1010fd8a09b8f4a7e0da67fe64cfb1d023bfb500d7ebpeter chang */ 1011d01612f3ae2515eb035d0c4ce954d8cb167a0a61Chris Mason if (td->o.odirect || td->o.mem_align || td->o.oatomic || 1012d01612f3ae2515eb035d0c4ce954d8cb167a0a61Chris Mason (td->io_ops->flags & FIO_RAWIO)) 1013fd8a09b8f4a7e0da67fe64cfb1d023bfb500d7ebpeter chang td->orig_buffer_size += page_mask + td->o.mem_align; 1014fd8a09b8f4a7e0da67fe64cfb1d023bfb500d7ebpeter chang 10152e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->o.mem_type == MEM_SHMHUGE || td->o.mem_type == MEM_MMAPHUGE) { 10162e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe unsigned long bs; 10172e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 10182e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe bs = td->orig_buffer_size + td->o.hugepage_size - 1; 10192e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->orig_buffer_size = bs & ~(td->o.hugepage_size - 1); 10202e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 10212e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 10222e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->orig_buffer_size != (size_t) td->orig_buffer_size) { 10232e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_err("fio: IO memory too large. Reduce max_bs or iodepth\n"); 10242e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 10252e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 10262e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 102759d8d0f5fb939ef23e4aa495dbe8fabd59d5bf25Jens Axboe if (data_xfer && allocate_io_mem(td)) 10282e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 10292e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1030d01612f3ae2515eb035d0c4ce954d8cb167a0a61Chris Mason if (td->o.odirect || td->o.mem_align || td->o.oatomic || 10312e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe (td->io_ops->flags & FIO_RAWIO)) 10322e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe p = PAGE_ALIGN(td->orig_buffer) + td->o.mem_align; 10332e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe else 10342e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe p = td->orig_buffer; 10352e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 10362e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe cl_align = os_cache_line_size(); 10372e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 10382e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe for (i = 0; i < max_units; i++) { 10392e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe void *ptr; 10402e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 10412e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->terminate) 10422e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 10432e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 10442e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ptr = fio_memalign(cl_align, sizeof(*io_u)); 10452e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!ptr) { 10462e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_err("fio: unable to allocate aligned memory\n"); 10472e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 10482e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 10492e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 10502e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe io_u = ptr; 10512e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe memset(io_u, 0, sizeof(*io_u)); 10522ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe INIT_FLIST_HEAD(&io_u->verify_list); 10532e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe dprint(FD_MEM, "io_u alloc %p, index %u\n", io_u, i); 10542e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 105559d8d0f5fb939ef23e4aa495dbe8fabd59d5bf25Jens Axboe if (data_xfer) { 10562e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe io_u->buf = p; 10572e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe dprint(FD_MEM, "io_u %p, mem %p\n", io_u, io_u->buf); 10582e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 10592e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td_write(td)) 10609c42684e32325da26e862280388798343c5f1305Jens Axboe io_u_fill_buffer(td, io_u, min_write, max_bs); 10612e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td_write(td) && td->o.verify_pattern_bytes) { 10622e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 10632e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * Fill the buffer with the pattern if we are 10642e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * going to be doing writes. 10652e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 1066ce35b1ece06ce27ebff111e47c2a6610c311a92bJens Axboe fill_verify_pattern(td, io_u->buf, max_bs, io_u, 0, 0); 10672e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 10682e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 10692e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 10702e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe io_u->index = i; 10712e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe io_u->flags = IO_U_F_FREE; 10722ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe io_u_qpush(&td->io_u_freelist, io_u); 10732ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe 10742ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe /* 10752ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe * io_u never leaves this stack, used for iteration of all 10762ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe * io_u buffers. 10772ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe */ 10782ae0b204743d6b4048c6fffd46c6280a70f2ecd1Jens Axboe io_u_qpush(&td->io_u_all, io_u); 1079c73ed24623ad94a5254e8302298797aba2eb1e9aJens Axboe 1080c73ed24623ad94a5254e8302298797aba2eb1e9aJens Axboe if (td->io_ops->io_u_init) { 1081c73ed24623ad94a5254e8302298797aba2eb1e9aJens Axboe int ret = td->io_ops->io_u_init(td, io_u); 1082c73ed24623ad94a5254e8302298797aba2eb1e9aJens Axboe 1083c73ed24623ad94a5254e8302298797aba2eb1e9aJens Axboe if (ret) { 1084c73ed24623ad94a5254e8302298797aba2eb1e9aJens Axboe log_err("fio: failed to init engine data: %d\n", ret); 1085c73ed24623ad94a5254e8302298797aba2eb1e9aJens Axboe return 1; 1086c73ed24623ad94a5254e8302298797aba2eb1e9aJens Axboe } 1087c73ed24623ad94a5254e8302298797aba2eb1e9aJens Axboe } 1088c73ed24623ad94a5254e8302298797aba2eb1e9aJens Axboe 10892e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe p += max_bs; 10902e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 10912e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 10922e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 0; 10932e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 10942e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 10952e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic int switch_ioscheduler(struct thread_data *td) 10962e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 10972e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe char tmp[256], tmp2[128]; 10982e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe FILE *f; 10992e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int ret; 11002e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 11012e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->io_ops->flags & FIO_DISKLESSIO) 11022e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 0; 11032e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 11042e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe sprintf(tmp, "%s/queue/scheduler", td->sysfs_root); 11052e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 11062e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe f = fopen(tmp, "r+"); 11072e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!f) { 11082e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (errno == ENOENT) { 11092e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_err("fio: os or kernel doesn't support IO scheduler" 11102e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe " switching\n"); 11112e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 0; 11122e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 11132e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_verror(td, errno, "fopen iosched"); 11142e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 11152e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 11162e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 11172e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 11182e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * Set io scheduler. 11192e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 11202e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ret = fwrite(td->o.ioscheduler, strlen(td->o.ioscheduler), 1, f); 11212e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (ferror(f) || ret != 1) { 11222e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_verror(td, errno, "fwrite"); 11232e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fclose(f); 11242e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 11252e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 11262e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 11272e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe rewind(f); 11282e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 11292e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 11302e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * Read back and check that the selected scheduler is now the default. 11312e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 113249c6f33d1739f96c16a0ab54cde3dc4ef64b90e9Jens Axboe ret = fread(tmp, sizeof(tmp), 1, f); 11332e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (ferror(f) || ret < 0) { 11342e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_verror(td, errno, "fread"); 11352e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fclose(f); 11362e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 11372e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 113849c6f33d1739f96c16a0ab54cde3dc4ef64b90e9Jens Axboe tmp[sizeof(tmp) - 1] = '\0'; 113949c6f33d1739f96c16a0ab54cde3dc4ef64b90e9Jens Axboe 11402e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 11412e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe sprintf(tmp2, "[%s]", td->o.ioscheduler); 11422e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!strstr(tmp, tmp2)) { 11432e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_err("fio: io scheduler %s not found\n", td->o.ioscheduler); 11442e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_verror(td, EINVAL, "iosched_switch"); 11452e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fclose(f); 11462e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 11472e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 11482e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 11492e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fclose(f); 11502e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 0; 11512e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 11522e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 11532e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic int keep_running(struct thread_data *td) 11542e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 115577731b29c35286a43ca6109ebf13389169ec20e9Jens Axboe unsigned long long limit; 115677731b29c35286a43ca6109ebf13389169ec20e9Jens Axboe 11572e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->done) 11582e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 0; 11592e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->o.time_based) 11602e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 11612e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->o.loops) { 11622e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->o.loops--; 11632e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 11642e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 11653939fe853fd146378811a05b78731d7f20d0152fJens Axboe if (exceeds_number_ios(td)) 11663939fe853fd146378811a05b78731d7f20d0152fJens Axboe return 0; 116726251d8d635bc9b9d31de2427095684b0c87b836Jens Axboe 116877731b29c35286a43ca6109ebf13389169ec20e9Jens Axboe if (td->o.io_limit) 116977731b29c35286a43ca6109ebf13389169ec20e9Jens Axboe limit = td->o.io_limit; 117077731b29c35286a43ca6109ebf13389169ec20e9Jens Axboe else 117177731b29c35286a43ca6109ebf13389169ec20e9Jens Axboe limit = td->o.size; 117277731b29c35286a43ca6109ebf13389169ec20e9Jens Axboe 117377731b29c35286a43ca6109ebf13389169ec20e9Jens Axboe if (limit != -1ULL && ddir_rw_sum(td->io_bytes) < limit) { 11745bd5f71a3e54950fb4bee64c7ad5edc67e40b8c8Jens Axboe uint64_t diff; 11755bd5f71a3e54950fb4bee64c7ad5edc67e40b8c8Jens Axboe 11765bd5f71a3e54950fb4bee64c7ad5edc67e40b8c8Jens Axboe /* 11775bd5f71a3e54950fb4bee64c7ad5edc67e40b8c8Jens Axboe * If the difference is less than the minimum IO size, we 11785bd5f71a3e54950fb4bee64c7ad5edc67e40b8c8Jens Axboe * are done. 11795bd5f71a3e54950fb4bee64c7ad5edc67e40b8c8Jens Axboe */ 118077731b29c35286a43ca6109ebf13389169ec20e9Jens Axboe diff = limit - ddir_rw_sum(td->io_bytes); 118174f4b020b3c8053ff319c3da900a5cf07e51c8ebJens Axboe if (diff < td_max_bs(td)) 11825bd5f71a3e54950fb4bee64c7ad5edc67e40b8c8Jens Axboe return 0; 11835bd5f71a3e54950fb4bee64c7ad5edc67e40b8c8Jens Axboe 1184002fe73409d1e3d5e7dfe2885f75885bfaf506bcJens Axboe if (fio_files_done(td)) 1185002fe73409d1e3d5e7dfe2885f75885bfaf506bcJens Axboe return 0; 1186002fe73409d1e3d5e7dfe2885f75885bfaf506bcJens Axboe 11872e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 11885bd5f71a3e54950fb4bee64c7ad5edc67e40b8c8Jens Axboe } 11892e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 11902e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 0; 11912e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 11922e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1193ce4864950d2ced54e0a1ccb8ef9a1e62135f1d2dErwan Velustatic int exec_string(struct thread_options *o, const char *string, const char *mode) 11942e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 1195ce4864950d2ced54e0a1ccb8ef9a1e62135f1d2dErwan Velu int ret, newlen = strlen(string) + strlen(o->name) + strlen(mode) + 9 + 1; 11962e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe char *str; 11972e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 11982e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe str = malloc(newlen); 1199ce4864950d2ced54e0a1ccb8ef9a1e62135f1d2dErwan Velu sprintf(str, "%s &> %s.%s.txt", string, o->name, mode); 12002e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1201ce4864950d2ced54e0a1ccb8ef9a1e62135f1d2dErwan Velu log_info("%s : Saving output of %s in %s.%s.txt\n",o->name, mode, o->name, mode); 12022e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ret = system(str); 12032e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (ret == -1) 12042e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_err("fio: exec of cmd <%s> failed\n", str); 12052e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 12062e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe free(str); 12072e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return ret; 12082e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 12092e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 12102e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe/* 1211621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse * Dry run to compute correct state of numberio for verification. 1212621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse */ 1213621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Cassestatic uint64_t do_dry_run(struct thread_data *td) 1214621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse{ 1215621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse uint64_t bytes_done[DDIR_RWDIR_CNT] = { 0, 0, 0 }; 1216621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse 1217621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse td_set_runstate(td, TD_RUNNING); 1218621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse 1219621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse while ((td->o.read_iolog_file && !flist_empty(&td->io_log_list)) || 1220621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse (!flist_empty(&td->trim_list)) || !io_bytes_exceeded(td)) { 1221621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse struct io_u *io_u; 1222621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse int ret; 1223621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse 1224621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse if (td->terminate || td->done) 1225621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse break; 1226621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse 1227621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse io_u = get_io_u(td); 1228621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse if (!io_u) 1229621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse break; 1230621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse 1231621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse io_u->flags |= IO_U_F_FLIGHT; 1232621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse io_u->error = 0; 1233621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse io_u->resid = 0; 1234621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse if (ddir_rw(acct_ddir(io_u))) 1235621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse td->io_issues[acct_ddir(io_u)]++; 1236621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse if (ddir_rw(io_u->ddir)) { 1237621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse io_u_mark_depth(td, 1); 1238621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse td->ts.total_io_u[io_u->ddir]++; 1239621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse } 1240621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse 12412e63e96b79c5153a32bec2d442f1c9a5790c17f5Puthikorn Voravootivat if (td_write(td) && io_u->ddir == DDIR_WRITE && 12422e63e96b79c5153a32bec2d442f1c9a5790c17f5Puthikorn Voravootivat td->o.do_verify && 12432e63e96b79c5153a32bec2d442f1c9a5790c17f5Puthikorn Voravootivat td->o.verify != VERIFY_NONE && 12442e63e96b79c5153a32bec2d442f1c9a5790c17f5Puthikorn Voravootivat !td->o.experimental_verify) 12452e63e96b79c5153a32bec2d442f1c9a5790c17f5Puthikorn Voravootivat log_io_piece(td, io_u); 12462e63e96b79c5153a32bec2d442f1c9a5790c17f5Puthikorn Voravootivat 1247621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse ret = io_u_sync_complete(td, io_u, bytes_done); 1248621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse (void) ret; 1249621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse } 1250621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse 1251621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse return bytes_done[DDIR_WRITE] + bytes_done[DDIR_TRIM]; 1252621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse} 1253621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse 1254621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse/* 12552e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * Entry point for the thread based jobs. The process based jobs end up 12562e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * here as well, after a little setup. 12572e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 12582e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic void *thread_main(void *data) 12592e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 12602e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe unsigned long long elapsed; 12612e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe struct thread_data *td = data; 12624896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe struct thread_options *o = &td->o; 12632e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe pthread_condattr_t attr; 12642e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int clear_state; 126528727df79348d4ebbd8fd4b0dd89bbca035f68d0Jens Axboe int ret; 12662e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 12674896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe if (!o->use_thread) { 12682e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe setsid(); 12692e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->pid = getpid(); 12702e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } else 12712e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->pid = gettid(); 12722e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1273334185e9e18c36b13c713ccdbb65e7ebd84795dbBruce Cran /* 1274334185e9e18c36b13c713ccdbb65e7ebd84795dbBruce Cran * fio_time_init() may not have been called yet if running as a server 1275334185e9e18c36b13c713ccdbb65e7ebd84795dbBruce Cran */ 1276334185e9e18c36b13c713ccdbb65e7ebd84795dbBruce Cran fio_time_init(); 1277334185e9e18c36b13c713ccdbb65e7ebd84795dbBruce Cran 12784896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe fio_local_clock_init(o->use_thread); 12795d879392c0e4025e508cce8118f6fc13ee57cc0eJens Axboe 12802e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe dprint(FD_PROCESS, "jobs pid=%d started\n", (int) td->pid); 12812e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1282122c772599f1b0a3148a5790775698d3fa92cc10Jens Axboe if (is_backend) 1283122c772599f1b0a3148a5790775698d3fa92cc10Jens Axboe fio_server_send_start(td); 1284122c772599f1b0a3148a5790775698d3fa92cc10Jens Axboe 12852e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe INIT_FLIST_HEAD(&td->io_log_list); 12862e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe INIT_FLIST_HEAD(&td->io_hist_list); 12872e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe INIT_FLIST_HEAD(&td->verify_list); 12882e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe INIT_FLIST_HEAD(&td->trim_list); 12891ae83d45ed853cd73b95b89ae14cacac5b97187eJens Axboe INIT_FLIST_HEAD(&td->next_rand_list); 12902e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe pthread_mutex_init(&td->io_u_lock, NULL); 12912e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->io_hist_tree = RB_ROOT; 12922e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 12932e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe pthread_condattr_init(&attr); 12942e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe pthread_cond_init(&td->verify_cond, &attr); 12952e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe pthread_cond_init(&td->free_cond, &attr); 12962e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 12972e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_set_runstate(td, TD_INITIALIZED); 12982e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe dprint(FD_MUTEX, "up startup_mutex\n"); 12992e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_mutex_up(startup_mutex); 13002e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe dprint(FD_MUTEX, "wait on td->mutex\n"); 13012e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_mutex_down(td->mutex); 13022e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe dprint(FD_MUTEX, "done waiting on td->mutex\n"); 13032e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 13042e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 13052e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * A new gid requires privilege, so we need to do this before setting 13062e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * the uid. 13072e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 13084896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe if (o->gid != -1U && setgid(o->gid)) { 13092e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_verror(td, errno, "setgid"); 13102e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe goto err; 13112e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 13124896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe if (o->uid != -1U && setuid(o->uid)) { 13132e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_verror(td, errno, "setuid"); 13142e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe goto err; 13152e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 13162e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 13172e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 13182e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * If we have a gettimeofday() thread, make sure we exclude that 13192e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * thread from this job 13202e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 13214896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe if (o->gtod_cpu) 13224896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe fio_cpu_clear(&o->cpumask, o->gtod_cpu); 13232e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 13242e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 13252e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * Set affinity first, in case it has an impact on the memory 13262e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * allocations. 13272e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 13284896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe if (o->cpumask_set) { 1329c2acfbac7e9eb8ac5407ac024c0fd000614c77e2Jens Axboe if (o->cpus_allowed_policy == FIO_CPUS_SPLIT) { 133030cb4c650a0a4c7ca79d4336ddb1fe3f988a4b2aJens Axboe ret = fio_cpus_split(&o->cpumask, td->thread_number - 1); 1331c2acfbac7e9eb8ac5407ac024c0fd000614c77e2Jens Axboe if (!ret) { 1332c2acfbac7e9eb8ac5407ac024c0fd000614c77e2Jens Axboe log_err("fio: no CPUs set\n"); 1333c2acfbac7e9eb8ac5407ac024c0fd000614c77e2Jens Axboe log_err("fio: Try increasing number of available CPUs\n"); 1334c2acfbac7e9eb8ac5407ac024c0fd000614c77e2Jens Axboe td_verror(td, EINVAL, "cpus_split"); 1335c2acfbac7e9eb8ac5407ac024c0fd000614c77e2Jens Axboe goto err; 1336c2acfbac7e9eb8ac5407ac024c0fd000614c77e2Jens Axboe } 1337c2acfbac7e9eb8ac5407ac024c0fd000614c77e2Jens Axboe } 133828727df79348d4ebbd8fd4b0dd89bbca035f68d0Jens Axboe ret = fio_setaffinity(td->pid, o->cpumask); 133928727df79348d4ebbd8fd4b0dd89bbca035f68d0Jens Axboe if (ret == -1) { 13404896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe td_verror(td, errno, "cpu_set_affinity"); 13414896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe goto err; 13424896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe } 13432e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 13442e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 134567bf982340d95ca98098ea050b54b4c7adb116c0Jens Axboe#ifdef CONFIG_LIBNUMA 1346d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren /* numa node setup */ 13474896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe if (o->numa_cpumask_set || o->numa_memmask_set) { 1348435228488ffd062f4eac710aaa862e04cd20dfeeDaniel Gollub struct bitmask *mask; 1349d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren int ret; 1350d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren 1351d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren if (numa_available() < 0) { 1352d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren td_verror(td, errno, "Does not support NUMA API\n"); 1353d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren goto err; 1354d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren } 1355d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren 13564896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe if (o->numa_cpumask_set) { 1357435228488ffd062f4eac710aaa862e04cd20dfeeDaniel Gollub mask = numa_parse_nodestring(o->numa_cpunodes); 1358435228488ffd062f4eac710aaa862e04cd20dfeeDaniel Gollub ret = numa_run_on_node_mask(mask); 1359435228488ffd062f4eac710aaa862e04cd20dfeeDaniel Gollub numa_free_nodemask(mask); 1360d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren if (ret == -1) { 1361d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren td_verror(td, errno, \ 1362d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren "numa_run_on_node_mask failed\n"); 1363d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren goto err; 1364d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren } 1365d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren } 1366d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren 13674896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe if (o->numa_memmask_set) { 1368d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren 1369435228488ffd062f4eac710aaa862e04cd20dfeeDaniel Gollub mask = NULL; 1370435228488ffd062f4eac710aaa862e04cd20dfeeDaniel Gollub if (o->numa_memnodes) 1371435228488ffd062f4eac710aaa862e04cd20dfeeDaniel Gollub mask = numa_parse_nodestring(o->numa_memnodes); 1372435228488ffd062f4eac710aaa862e04cd20dfeeDaniel Gollub 13734896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe switch (o->numa_mem_mode) { 1374d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren case MPOL_INTERLEAVE: 1375435228488ffd062f4eac710aaa862e04cd20dfeeDaniel Gollub numa_set_interleave_mask(mask); 1376d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren break; 1377d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren case MPOL_BIND: 1378435228488ffd062f4eac710aaa862e04cd20dfeeDaniel Gollub numa_set_membind(mask); 1379d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren break; 1380d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren case MPOL_LOCAL: 1381d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren numa_set_localalloc(); 1382d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren break; 1383d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren case MPOL_PREFERRED: 13844896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe numa_set_preferred(o->numa_mem_prefer_node); 1385d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren break; 1386d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren case MPOL_DEFAULT: 1387d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren default: 1388d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren break; 1389d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren } 1390d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren 1391435228488ffd062f4eac710aaa862e04cd20dfeeDaniel Gollub if (mask) 1392435228488ffd062f4eac710aaa862e04cd20dfeeDaniel Gollub numa_free_nodemask(mask); 1393435228488ffd062f4eac710aaa862e04cd20dfeeDaniel Gollub 1394d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren } 1395d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren } 1396d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren#endif 1397d0b937ed8841484c2d651919943e8e4c70dfd460Yufei Ren 13989a3f110001f68968def40797bed5632c2406cdfaJens Axboe if (fio_pin_memory(td)) 13999a3f110001f68968def40797bed5632c2406cdfaJens Axboe goto err; 14009a3f110001f68968def40797bed5632c2406cdfaJens Axboe 14012e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 14022e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * May alter parameters that init_io_u() will use, so we need to 14032e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * do this first. 14042e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 14052e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (init_iolog(td)) 14062e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe goto err; 14072e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 14082e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (init_io_u(td)) 14092e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe goto err; 14102e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 14114896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe if (o->verify_async && verify_async_init(td)) 14122e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe goto err; 14132e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 141428727df79348d4ebbd8fd4b0dd89bbca035f68d0Jens Axboe if (o->ioprio) { 141528727df79348d4ebbd8fd4b0dd89bbca035f68d0Jens Axboe ret = ioprio_set(IOPRIO_WHO_PROCESS, 0, o->ioprio_class, o->ioprio); 141628727df79348d4ebbd8fd4b0dd89bbca035f68d0Jens Axboe if (ret == -1) { 14172e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_verror(td, errno, "ioprio_set"); 14182e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe goto err; 14192e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 14202e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 14212e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 14224896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe if (o->cgroup && cgroup_setup(td, cgroup_list, &cgroup_mnt)) 14232e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe goto err; 14242e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1425649c10c59b016ae8586e54746d3761bc6df33c9bBruce Cran errno = 0; 14264896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe if (nice(o->nice) == -1 && errno != 0) { 14272e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_verror(td, errno, "nice"); 14282e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe goto err; 14292e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 14302e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 14314896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe if (o->ioscheduler && switch_ioscheduler(td)) 14322e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe goto err; 14332e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 14344896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe if (!o->create_serialize && setup_files(td)) 14352e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe goto err; 14362e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 14372e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td_io_init(td)) 14382e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe goto err; 14392e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 14402e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (init_random_map(td)) 14412e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe goto err; 14422e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1443ce4864950d2ced54e0a1ccb8ef9a1e62135f1d2dErwan Velu if (o->exec_prerun && exec_string(o, o->exec_prerun, (const char *)"prerun")) 14444896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe goto err; 14452e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 14464896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe if (o->pre_read) { 14472e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (pre_read_files(td) < 0) 14482e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe goto err; 14492e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 14502e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1451dc5bfbb227f1af83adf0b686d147b7ae2107c144Jens Axboe fio_verify_init(td); 1452dc5bfbb227f1af83adf0b686d147b7ae2107c144Jens Axboe 14532e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_gettime(&td->epoch, NULL); 145444404c5a7f90aa42c3228b56b6f686f15a50fb29Jens Axboe fio_getrusage(&td->ru_start); 14552e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe clear_state = 0; 14562e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe while (keep_running(td)) { 1457100f49f105126a26de6f1069679c20fd75194188Jens Axboe uint64_t verify_bytes; 1458100f49f105126a26de6f1069679c20fd75194188Jens Axboe 14592e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_gettime(&td->start, NULL); 14602e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe memcpy(&td->bw_sample_time, &td->start, sizeof(td->start)); 14612e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe memcpy(&td->iops_sample_time, &td->start, sizeof(td->start)); 14622e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe memcpy(&td->tv_cache, &td->start, sizeof(td->start)); 14632e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 14644896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe if (o->ratemin[DDIR_READ] || o->ratemin[DDIR_WRITE] || 14654896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe o->ratemin[DDIR_TRIM]) { 14666eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li memcpy(&td->lastrate[DDIR_READ], &td->bw_sample_time, 14672e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe sizeof(td->bw_sample_time)); 14686eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li memcpy(&td->lastrate[DDIR_WRITE], &td->bw_sample_time, 14696eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li sizeof(td->bw_sample_time)); 14706eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li memcpy(&td->lastrate[DDIR_TRIM], &td->bw_sample_time, 14712e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe sizeof(td->bw_sample_time)); 14722e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 14732e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 14742e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (clear_state) 14752e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe clear_io_state(td); 14762e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 14772e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe prune_io_piece_log(td); 14782e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1479621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse if (td->o.verify_only && (td_write(td) || td_rw(td))) 1480621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse verify_bytes = do_dry_run(td); 1481621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse else 1482621677626f2551bedfdc4a5fc3b3e5f8492b94faJuan Casse verify_bytes = do_io(td); 14832e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 14842e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe clear_state = 1; 14852e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 14862e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td_read(td) && td->io_bytes[DDIR_READ]) { 14872e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe elapsed = utime_since_now(&td->start); 14882e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->ts.runtime[DDIR_READ] += elapsed; 14892e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 14902e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td_write(td) && td->io_bytes[DDIR_WRITE]) { 14912e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe elapsed = utime_since_now(&td->start); 14922e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->ts.runtime[DDIR_WRITE] += elapsed; 14932e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 14946eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li if (td_trim(td) && td->io_bytes[DDIR_TRIM]) { 14956eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li elapsed = utime_since_now(&td->start); 14966eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li td->ts.runtime[DDIR_TRIM] += elapsed; 14976eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li } 14982e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 14992e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->error || td->terminate) 15002e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 15012e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 15024896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe if (!o->do_verify || 15034896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe o->verify == VERIFY_NONE || 15042e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe (td->io_ops->flags & FIO_UNIDIR)) 15052e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe continue; 15062e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 15072e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe clear_io_state(td); 15082e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 15092e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_gettime(&td->start, NULL); 15102e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1511100f49f105126a26de6f1069679c20fd75194188Jens Axboe do_verify(td, verify_bytes); 15122e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 15132e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->ts.runtime[DDIR_READ] += utime_since_now(&td->start); 15142e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 15152e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->error || td->terminate) 15162e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 15172e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 15182e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 15192e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe update_rusage_stat(td); 15206eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li td->ts.runtime[DDIR_READ] = (td->ts.runtime[DDIR_READ] + 999) / 1000; 15216eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li td->ts.runtime[DDIR_WRITE] = (td->ts.runtime[DDIR_WRITE] + 999) / 1000; 15226eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li td->ts.runtime[DDIR_TRIM] = (td->ts.runtime[DDIR_TRIM] + 999) / 1000; 15232e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->ts.total_run_time = mtime_since_now(&td->epoch); 15246eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li td->ts.io_bytes[DDIR_READ] = td->io_bytes[DDIR_READ]; 15256eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li td->ts.io_bytes[DDIR_WRITE] = td->io_bytes[DDIR_WRITE]; 15266eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li td->ts.io_bytes[DDIR_TRIM] = td->io_bytes[DDIR_TRIM]; 15272e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 15289a3f110001f68968def40797bed5632c2406cdfaJens Axboe fio_unpin_memory(td); 15299a3f110001f68968def40797bed5632c2406cdfaJens Axboe 1530905e3d4ff8d01a4402186e1ad1f5f1fbbb6b7ef4Jens Axboe fio_writeout_logs(td); 15312e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 15324896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe if (o->exec_postrun) 1533ce4864950d2ced54e0a1ccb8ef9a1e62135f1d2dErwan Velu exec_string(o, o->exec_postrun, (const char *)"postrun"); 15342e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 15352e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (exitall_on_terminate) 15362e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_terminate_threads(td->groupid); 15372e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 15382e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboeerr: 15392e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->error) 15402e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_info("fio: pid=%d, err=%d/%s\n", (int) td->pid, td->error, 15412e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->verror); 15422e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 15434896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe if (o->verify_async) 15442e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe verify_async_exit(td); 15452e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 15462e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe close_and_free_files(td); 15472e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe cleanup_io_u(td); 154832dbca2c33c37dcd62bc118dbea33f3c0a488c82Bruce Cran close_ioengine(td); 15492e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe cgroup_shutdown(td, &cgroup_mnt); 15502e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 15514896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe if (o->cpumask_set) { 15524896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe int ret = fio_cpuset_exit(&o->cpumask); 15532e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 15542e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_verror(td, ret, "fio_cpuset_exit"); 15552e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 15562e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 15572e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 15582e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * do this very late, it will log file closing as well 15592e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 15604896473eb0c69d870c41a6a679765d2d5f03e199Jens Axboe if (o->write_iolog_file) 15612e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe write_iolog_close(td); 15622e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1563c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe fio_mutex_remove(td->rusage_sem); 1564c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe td->rusage_sem = NULL; 1565c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe 1566ea66e04fe1a803f6a9ddf31cb999641d4396d67cJens Axboe fio_mutex_remove(td->mutex); 1567ea66e04fe1a803f6a9ddf31cb999641d4396d67cJens Axboe td->mutex = NULL; 1568ea66e04fe1a803f6a9ddf31cb999641d4396d67cJens Axboe 15692e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_set_runstate(td, TD_EXITED); 1570e43606c2b13ad7fc1af2bbe4a61cf8480ee3a532Bruce Cran return (void *) (uintptr_t) td->error; 15712e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 15722e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 15732e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 15742e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe/* 15752e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * We cannot pass the td data into a forked process, so attach the td and 15762e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * pass it to the thread worker. 15772e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 15782e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic int fork_main(int shmid, int offset) 15792e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 15802e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe struct thread_data *td; 15812e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe void *data, *ret; 15822e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 15832e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#ifndef __hpux 15842e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe data = shmat(shmid, NULL, 0); 15852e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (data == (void *) -1) { 15862e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int __err = errno; 15872e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 15882e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe perror("shmat"); 15892e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return __err; 15902e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 15912e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#else 15922e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 15932e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * HP-UX inherits shm mappings? 15942e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 15952e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe data = threads; 15962e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe#endif 15972e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 15982e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td = data + offset * sizeof(struct thread_data); 15992e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ret = thread_main(td); 16002e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe shmdt(data); 1601e43606c2b13ad7fc1af2bbe4a61cf8480ee3a532Bruce Cran return (int) (uintptr_t) ret; 16022e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 16032e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 16042e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe/* 16052e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * Run over the job map and reap the threads that have exited, if any. 16062e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 16072e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic void reap_threads(unsigned int *nr_running, unsigned int *t_rate, 16082e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe unsigned int *m_rate) 16092e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 16102e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe struct thread_data *td; 16112e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe unsigned int cputhreads, realthreads, pending; 16122e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int i, status, ret; 16132e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 16142e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 16152e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * reap exited threads (TD_EXITED -> TD_REAPED) 16162e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 16172e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe realthreads = pending = cputhreads = 0; 16182e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe for_each_td(td, i) { 16192e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int flags = 0; 16202e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 16212e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 16222e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * ->io_ops is NULL for a thread that has closed its 16232e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * io engine 16242e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 16252e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->io_ops && !strcmp(td->io_ops->name, "cpuio")) 16262e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe cputhreads++; 16272e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe else 16282e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe realthreads++; 16292e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 16302e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!td->pid) { 16312e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe pending++; 16322e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe continue; 16332e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 16342e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->runstate == TD_REAPED) 16352e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe continue; 16362e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->o.use_thread) { 16372e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->runstate == TD_EXITED) { 16382e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_set_runstate(td, TD_REAPED); 16392e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe goto reaped; 16402e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 16412e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe continue; 16422e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 16432e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 16442e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe flags = WNOHANG; 16452e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->runstate == TD_EXITED) 16462e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe flags = 0; 16472e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 16482e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 16492e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * check if someone quit or got killed in an unusual way 16502e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 16512e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ret = waitpid(td->pid, &status, flags); 16522e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (ret < 0) { 16532e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (errno == ECHILD) { 16542e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_err("fio: pid=%d disappeared %d\n", 16552e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe (int) td->pid, td->runstate); 1656a5e371a61c9e37a2389f7473d65e35b29dec1ccdJens Axboe td->sig = ECHILD; 16572e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_set_runstate(td, TD_REAPED); 16582e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe goto reaped; 16592e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 16602e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe perror("waitpid"); 16612e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } else if (ret == td->pid) { 16622e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (WIFSIGNALED(status)) { 16632e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int sig = WTERMSIG(status); 16642e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 166536d80bc7c7f7fbc2612941b7dd7ceaf645798c7fJens Axboe if (sig != SIGTERM && sig != SIGUSR2) 16662e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_err("fio: pid=%d, got signal=%d\n", 16672e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe (int) td->pid, sig); 1668a5e371a61c9e37a2389f7473d65e35b29dec1ccdJens Axboe td->sig = sig; 16692e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_set_runstate(td, TD_REAPED); 16702e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe goto reaped; 16712e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 16722e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (WIFEXITED(status)) { 16732e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (WEXITSTATUS(status) && !td->error) 16742e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->error = WEXITSTATUS(status); 16752e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 16762e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_set_runstate(td, TD_REAPED); 16772e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe goto reaped; 16782e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 16792e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 16802e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 16812e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 16822e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * thread is not dead, continue 16832e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 16842e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe pending++; 16852e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe continue; 16862e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboereaped: 16872e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe (*nr_running)--; 1688342f4be43b2e74cf2f9a2102afdad11c8951b73fJens Axboe (*m_rate) -= ddir_rw_sum(td->o.ratemin); 1689342f4be43b2e74cf2f9a2102afdad11c8951b73fJens Axboe (*t_rate) -= ddir_rw_sum(td->o.rate); 16902e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!td->pid) 16912e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe pending--; 16922e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 16932e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->error) 16942e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe exit_value++; 16952e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 16962e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe done_secs += mtime_since_now(&td->epoch) / 1000; 16974a88752ad8e0b67ef9499262998d221559d6dac1Jens Axboe profile_td_exit(td); 16982e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 16992e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 17002e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (*nr_running == cputhreads && !pending && realthreads) 17012e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_terminate_threads(TERMINATE_ALL); 17022e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 17032e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 170406464907159baf7a1eeac654a023743e33f28d86Jens Axboestatic void do_usleep(unsigned int usecs) 170506464907159baf7a1eeac654a023743e33f28d86Jens Axboe{ 170606464907159baf7a1eeac654a023743e33f28d86Jens Axboe check_for_running_stats(); 170706464907159baf7a1eeac654a023743e33f28d86Jens Axboe usleep(usecs); 170806464907159baf7a1eeac654a023743e33f28d86Jens Axboe} 170906464907159baf7a1eeac654a023743e33f28d86Jens Axboe 17102e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe/* 17112e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * Main function for kicking off and reaping jobs, as needed. 17122e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 17132e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic void run_threads(void) 17142e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 17152e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe struct thread_data *td; 17162e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe unsigned int i, todo, nr_running, m_rate, t_rate, nr_started; 17170de5b26f6e177aacac0683306c47e0cbaf58b0b6Jens Axboe uint64_t spent; 17182e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 17192e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (fio_gtod_offload && fio_start_gtod_thread()) 17202e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return; 1721334185e9e18c36b13c713ccdbb65e7ebd84795dbBruce Cran 1722f2a2ce0eedb44eaa8689e4cbfa77c79b1751b216Huadong Liu fio_idle_prof_init(); 17232e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 17242e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe set_sig_handlers(); 17252e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 17263a5f6bde1d14faa9e60cde1c8b45c101bc462f34Jens Axboe nr_thread = nr_process = 0; 17273a5f6bde1d14faa9e60cde1c8b45c101bc462f34Jens Axboe for_each_td(td, i) { 17283a5f6bde1d14faa9e60cde1c8b45c101bc462f34Jens Axboe if (td->o.use_thread) 17293a5f6bde1d14faa9e60cde1c8b45c101bc462f34Jens Axboe nr_thread++; 17303a5f6bde1d14faa9e60cde1c8b45c101bc462f34Jens Axboe else 17313a5f6bde1d14faa9e60cde1c8b45c101bc462f34Jens Axboe nr_process++; 17323a5f6bde1d14faa9e60cde1c8b45c101bc462f34Jens Axboe } 17333a5f6bde1d14faa9e60cde1c8b45c101bc462f34Jens Axboe 1734f3afa57e36550288340f1b6c694f354ae72654b9Jens Axboe if (output_format == FIO_OUTPUT_NORMAL) { 17352e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_info("Starting "); 17362e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (nr_thread) 17372e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_info("%d thread%s", nr_thread, 17382e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe nr_thread > 1 ? "s" : ""); 17392e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (nr_process) { 17402e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (nr_thread) 17412e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_info(" and "); 17422e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_info("%d process%s", nr_process, 17432e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe nr_process > 1 ? "es" : ""); 17442e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 17452e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_info("\n"); 17462e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fflush(stdout); 17472e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 17482e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 17492e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe todo = thread_number; 17502e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe nr_running = 0; 17512e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe nr_started = 0; 17522e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe m_rate = t_rate = 0; 17532e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 17542e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe for_each_td(td, i) { 17552e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe print_status_init(td->thread_number - 1); 17562e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 17572e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!td->o.create_serialize) 17582e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe continue; 17592e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 17602e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 17612e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * do file setup here so it happens sequentially, 17622e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * we don't want X number of threads getting their 17632e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * client data interspersed on disk 17642e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 17652e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (setup_files(td)) { 17662e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe exit_value++; 17672e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->error) 17682e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_err("fio: pid=%d, err=%d/%s\n", 17692e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe (int) td->pid, td->error, td->verror); 17702e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_set_runstate(td, TD_REAPED); 17712e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe todo--; 17722e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } else { 17732e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe struct fio_file *f; 17742e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe unsigned int j; 17752e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 17762e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 17772e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * for sharing to work, each job must always open 17782e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * its own files. so close them, if we opened them 17792e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * for creation 17802e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 17812e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe for_each_file(td, f, j) { 17822e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (fio_file_open(f)) 17832e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_io_close_file(td, f); 17842e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 17852e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 17862e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 17872e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1788f2a2ce0eedb44eaa8689e4cbfa77c79b1751b216Huadong Liu /* start idle threads before io threads start to run */ 1789f2a2ce0eedb44eaa8689e4cbfa77c79b1751b216Huadong Liu fio_idle_prof_start(); 1790f2a2ce0eedb44eaa8689e4cbfa77c79b1751b216Huadong Liu 17912e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe set_genesis_time(); 17922e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 17932e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe while (todo) { 17942e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe struct thread_data *map[REAL_MAX_JOBS]; 17952e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe struct timeval this_start; 17962e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int this_jobs = 0, left; 17972e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 17982e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 17992e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * create threads (TD_NOT_CREATED -> TD_CREATED) 18002e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 18012e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe for_each_td(td, i) { 18022e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->runstate != TD_NOT_CREATED) 18032e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe continue; 18042e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 18052e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 18062e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * never got a chance to start, killed by other 18072e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * thread for some reason 18082e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 18092e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->terminate) { 18102e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe todo--; 18112e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe continue; 18122e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 18132e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 18142e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->o.start_delay) { 18150de5b26f6e177aacac0683306c47e0cbaf58b0b6Jens Axboe spent = utime_since_genesis(); 18162e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 181774454ce40f1a5e1e682da0a8acb824a7f6910270Christian Ehrhardt if (td->o.start_delay > spent) 18182e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe continue; 18192e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 18202e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 18212e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->o.stonewall && (nr_started || nr_running)) { 18222e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe dprint(FD_PROCESS, "%s: stonewall wait\n", 18232e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td->o.name); 18242e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 18252e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 18262e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 18272e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe init_disk_util(td); 18282e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1829c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe td->rusage_sem = fio_mutex_init(FIO_MUTEX_LOCKED); 1830c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe td->update_rusage = 0; 1831c97f1ad6d2a2fb4fe9f3b15e40158aac21e5699eJens Axboe 18322e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 18332e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * Set state to created. Thread will transition 18342e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * to TD_INITIALIZED when it's done setting up. 18352e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 18362e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_set_runstate(td, TD_CREATED); 18372e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe map[this_jobs++] = td; 18382e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe nr_started++; 18392e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 18402e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->o.use_thread) { 18412e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int ret; 18422e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 18432e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe dprint(FD_PROCESS, "will pthread_create\n"); 18442e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ret = pthread_create(&td->thread, NULL, 18452e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe thread_main, td); 18462e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (ret) { 18472e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_err("pthread_create: %s\n", 18482e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe strerror(ret)); 18492e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe nr_started--; 18502e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 18512e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 18522e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ret = pthread_detach(td->thread); 18532e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (ret) 18542e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_err("pthread_detach: %s", 18552e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe strerror(ret)); 18562e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } else { 18572e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe pid_t pid; 18582e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe dprint(FD_PROCESS, "will fork\n"); 18592e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe pid = fork(); 18602e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!pid) { 18612e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int ret = fork_main(shm_id, i); 18622e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 18632e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe _exit(ret); 18642e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } else if (i == fio_debug_jobno) 18652e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe *fio_debug_jobp = pid; 18662e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 18672e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe dprint(FD_MUTEX, "wait on startup_mutex\n"); 18682e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (fio_mutex_down_timeout(startup_mutex, 10)) { 18692e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_err("fio: job startup hung? exiting.\n"); 18702e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_terminate_threads(TERMINATE_ALL); 18712e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_abort = 1; 18722e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe nr_started--; 18732e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 18742e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 18752e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe dprint(FD_MUTEX, "done waiting on startup_mutex\n"); 18762e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 18772e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 18782e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 18792e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * Wait for the started threads to transition to 18802e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * TD_INITIALIZED. 18812e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 18822e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_gettime(&this_start, NULL); 18832e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe left = this_jobs; 18842e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe while (left && !fio_abort) { 18852e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (mtime_since_now(&this_start) > JOB_START_TIMEOUT) 18862e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 18872e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 188806464907159baf7a1eeac654a023743e33f28d86Jens Axboe do_usleep(100000); 18892e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 18902e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe for (i = 0; i < this_jobs; i++) { 18912e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td = map[i]; 18922e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!td) 18932e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe continue; 18942e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->runstate == TD_INITIALIZED) { 18952e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe map[i] = NULL; 18962e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe left--; 18972e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } else if (td->runstate >= TD_EXITED) { 18982e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe map[i] = NULL; 18992e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe left--; 19002e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe todo--; 19012e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe nr_running++; /* work-around... */ 19022e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 19032e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 19042e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 19052e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 19062e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (left) { 19074e87c37a06b96c4ed1f941eb91ec25e839c68d21Jens Axboe log_err("fio: %d job%s failed to start\n", left, 19084e87c37a06b96c4ed1f941eb91ec25e839c68d21Jens Axboe left > 1 ? "s" : ""); 19092e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe for (i = 0; i < this_jobs; i++) { 19102e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td = map[i]; 19112e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!td) 19122e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe continue; 19132e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe kill(td->pid, SIGTERM); 19142e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 19152e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 19162e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 19172e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 19182e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe /* 19192e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe * start created threads (TD_INITIALIZED -> TD_RUNNING). 19202e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe */ 19212e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe for_each_td(td, i) { 19222e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (td->runstate != TD_INITIALIZED) 19232e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe continue; 19242e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 19252e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (in_ramp_time(td)) 19262e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_set_runstate(td, TD_RAMP); 19272e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe else 19282e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe td_set_runstate(td, TD_RUNNING); 19292e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe nr_running++; 19302e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe nr_started--; 1931342f4be43b2e74cf2f9a2102afdad11c8951b73fJens Axboe m_rate += ddir_rw_sum(td->o.ratemin); 1932342f4be43b2e74cf2f9a2102afdad11c8951b73fJens Axboe t_rate += ddir_rw_sum(td->o.rate); 19332e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe todo--; 19342e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_mutex_up(td->mutex); 19352e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 19362e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 19372e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe reap_threads(&nr_running, &t_rate, &m_rate); 19382e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1939122c772599f1b0a3148a5790775698d3fa92cc10Jens Axboe if (todo) 194006464907159baf7a1eeac654a023743e33f28d86Jens Axboe do_usleep(100000); 19412e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 19422e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 19432e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe while (nr_running) { 19442e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe reap_threads(&nr_running, &t_rate, &m_rate); 194506464907159baf7a1eeac654a023743e33f28d86Jens Axboe do_usleep(10000); 19462e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 19472e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 1948f2a2ce0eedb44eaa8689e4cbfa77c79b1751b216Huadong Liu fio_idle_prof_stop(); 1949f2a2ce0eedb44eaa8689e4cbfa77c79b1751b216Huadong Liu 19502e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe update_io_ticks(); 19512e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 19522e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 19539ec7779f89d4dde9fe1e23d68abaefda54b35de7Jens Axboevoid wait_for_disk_thread_exit(void) 19549ec7779f89d4dde9fe1e23d68abaefda54b35de7Jens Axboe{ 19559ec7779f89d4dde9fe1e23d68abaefda54b35de7Jens Axboe fio_mutex_down(disk_thread_mutex); 19569ec7779f89d4dde9fe1e23d68abaefda54b35de7Jens Axboe} 19579ec7779f89d4dde9fe1e23d68abaefda54b35de7Jens Axboe 195827357187b0899380bae5c6748f2b2d034502e162Jens Axboestatic void free_disk_util(void) 195927357187b0899380bae5c6748f2b2d034502e162Jens Axboe{ 196027357187b0899380bae5c6748f2b2d034502e162Jens Axboe disk_util_start_exit(); 196127357187b0899380bae5c6748f2b2d034502e162Jens Axboe wait_for_disk_thread_exit(); 196227357187b0899380bae5c6748f2b2d034502e162Jens Axboe disk_util_prune_entries(); 196327357187b0899380bae5c6748f2b2d034502e162Jens Axboe} 196427357187b0899380bae5c6748f2b2d034502e162Jens Axboe 19652e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic void *disk_thread_main(void *data) 19662e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 19679ec7779f89d4dde9fe1e23d68abaefda54b35de7Jens Axboe int ret = 0; 19689ec7779f89d4dde9fe1e23d68abaefda54b35de7Jens Axboe 19692e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_mutex_up(startup_mutex); 19702e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 19719ec7779f89d4dde9fe1e23d68abaefda54b35de7Jens Axboe while (threads && !ret) { 19722e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe usleep(DISK_UTIL_MSEC * 1000); 19732e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!threads) 19742e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe break; 19759ec7779f89d4dde9fe1e23d68abaefda54b35de7Jens Axboe ret = update_io_ticks(); 19762e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 19772e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!is_backend) 19782e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe print_thread_status(); 19792e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 19802e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 19819ec7779f89d4dde9fe1e23d68abaefda54b35de7Jens Axboe fio_mutex_up(disk_thread_mutex); 19822e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return NULL; 19832e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 19842e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 19852e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboestatic int create_disk_util_thread(void) 19862e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 19872e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int ret; 19882e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 19899ec7779f89d4dde9fe1e23d68abaefda54b35de7Jens Axboe setup_disk_util(); 19909ec7779f89d4dde9fe1e23d68abaefda54b35de7Jens Axboe 1991521da527743088a9bd2ab882f8b64799d49d5848Jens Axboe disk_thread_mutex = fio_mutex_init(FIO_MUTEX_LOCKED); 19929ec7779f89d4dde9fe1e23d68abaefda54b35de7Jens Axboe 19932e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ret = pthread_create(&disk_util_thread, NULL, disk_thread_main, NULL); 19942e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (ret) { 19959ec7779f89d4dde9fe1e23d68abaefda54b35de7Jens Axboe fio_mutex_remove(disk_thread_mutex); 19962e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_err("Can't create disk util thread: %s\n", strerror(ret)); 19972e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 19982e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 19992e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 20002e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe ret = pthread_detach(disk_util_thread); 20012e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (ret) { 20029ec7779f89d4dde9fe1e23d68abaefda54b35de7Jens Axboe fio_mutex_remove(disk_thread_mutex); 20032e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe log_err("Can't detatch disk util thread: %s\n", strerror(ret)); 20042e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 20052e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 20062e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 20072e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe dprint(FD_MUTEX, "wait on startup_mutex\n"); 20082e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_mutex_down(startup_mutex); 20092e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe dprint(FD_MUTEX, "done waiting on startup_mutex\n"); 20102e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 0; 20112e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 20122e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 20132e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboeint fio_backend(void) 20142e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe{ 20152e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe struct thread_data *td; 20162e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe int i; 20172e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 20182e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (exec_profile) { 20192e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (load_profile(exec_profile)) 20202e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 20212e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe free(exec_profile); 20222e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe exec_profile = NULL; 20232e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 20242e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!thread_number) 20252e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 0; 20262e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 20272e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (write_bw_log) { 20285a812f9b9d987d1cacbb6f17a65835cbe0d66361Jens Axboe setup_log(&agg_io_log[DDIR_READ], 0, IO_LOG_TYPE_BW); 20295a812f9b9d987d1cacbb6f17a65835cbe0d66361Jens Axboe setup_log(&agg_io_log[DDIR_WRITE], 0, IO_LOG_TYPE_BW); 20305a812f9b9d987d1cacbb6f17a65835cbe0d66361Jens Axboe setup_log(&agg_io_log[DDIR_TRIM], 0, IO_LOG_TYPE_BW); 20312e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 20322e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 2033521da527743088a9bd2ab882f8b64799d49d5848Jens Axboe startup_mutex = fio_mutex_init(FIO_MUTEX_LOCKED); 20342e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (startup_mutex == NULL) 20352e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return 1; 20362e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 20372e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe set_genesis_time(); 2038cef9175e52e8a240117b9f45b228fdaa0f1c0572Jens Axboe stat_init(); 20392e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe create_disk_util_thread(); 20402e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 20412e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe cgroup_list = smalloc(sizeof(*cgroup_list)); 20422e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe INIT_FLIST_HEAD(cgroup_list); 20432e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 20442e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe run_threads(); 20452e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 20462e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (!fio_abort) { 20472e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe show_run_stats(); 20482e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe if (write_bw_log) { 20492e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe __finish_log(agg_io_log[DDIR_READ], "agg-read_bw.log"); 20502e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe __finish_log(agg_io_log[DDIR_WRITE], 20512e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe "agg-write_bw.log"); 20526eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li __finish_log(agg_io_log[DDIR_TRIM], 20536eaf09d6e9ca1f8accb057cdb18620b7e53ae33fShaohua Li "agg-write_bw.log"); 20542e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 20552e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe } 20562e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 20572e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe for_each_td(td, i) 20582e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_options_free(td); 20592e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 2060a462baefd211d1847cefbaa500c8aaad2128374bJens Axboe free_disk_util(); 20612e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe cgroup_kill(cgroup_list); 20622e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe sfree(cgroup_list); 20632e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe sfree(cgroup_mnt); 20642e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe 20652e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe fio_mutex_remove(startup_mutex); 20669ec7779f89d4dde9fe1e23d68abaefda54b35de7Jens Axboe fio_mutex_remove(disk_thread_mutex); 2067cef9175e52e8a240117b9f45b228fdaa0f1c0572Jens Axboe stat_exit(); 20682e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe return exit_value; 20692e1df07d1ea30e0304cc65370f3ed161a6f22cd4Jens Axboe} 2070