ioengines.c revision cb849a795d2bfd99efdd66f5a3e475c9fb3edd86
1/* 2 * The io parts of the fio tool, includes workers for sync and mmap'ed 3 * io, as well as both posix and linux libaio support. 4 * 5 * sync io is implemented on top of aio. 6 * 7 * This is not really specific to fio, if the get_io_u/put_io_u and 8 * structures was pulled into this as well it would be a perfectly 9 * generic io engine that could be used for other projects. 10 * 11 */ 12#include <stdio.h> 13#include <stdlib.h> 14#include <unistd.h> 15#include <string.h> 16#include <dlfcn.h> 17#include <assert.h> 18 19#include "fio.h" 20#include "diskutil.h" 21 22static FLIST_HEAD(engine_list); 23 24static int check_engine_ops(struct ioengine_ops *ops) 25{ 26 if (ops->version != FIO_IOOPS_VERSION) { 27 log_err("bad ioops version %d (want %d)\n", ops->version, 28 FIO_IOOPS_VERSION); 29 return 1; 30 } 31 32 if (!ops->queue) { 33 log_err("%s: no queue handler\n", ops->name); 34 return 1; 35 } 36 37 /* 38 * sync engines only need a ->queue() 39 */ 40 if (ops->flags & FIO_SYNCIO) 41 return 0; 42 43 if (!ops->event) { 44 log_err("%s: no event handler\n", ops->name); 45 return 1; 46 } 47 if (!ops->getevents) { 48 log_err("%s: no getevents handler\n", ops->name); 49 return 1; 50 } 51 if (!ops->queue) { 52 log_err("%s: no queue handler\n", ops->name); 53 return 1; 54 } 55 56 return 0; 57} 58 59void unregister_ioengine(struct ioengine_ops *ops) 60{ 61 dprint(FD_IO, "ioengine %s unregistered\n", ops->name); 62 flist_del(&ops->list); 63 INIT_FLIST_HEAD(&ops->list); 64} 65 66void register_ioengine(struct ioengine_ops *ops) 67{ 68 dprint(FD_IO, "ioengine %s registered\n", ops->name); 69 INIT_FLIST_HEAD(&ops->list); 70 flist_add_tail(&ops->list, &engine_list); 71} 72 73static struct ioengine_ops *find_ioengine(const char *name) 74{ 75 struct ioengine_ops *ops; 76 struct flist_head *entry; 77 78 flist_for_each(entry, &engine_list) { 79 ops = flist_entry(entry, struct ioengine_ops, list); 80 if (!strcmp(name, ops->name)) 81 return ops; 82 } 83 84 return NULL; 85} 86 87static struct ioengine_ops *dlopen_ioengine(struct thread_data *td, 88 const char *engine_lib) 89{ 90 struct ioengine_ops *ops; 91 void *dlhandle; 92 93 dprint(FD_IO, "dload engine %s\n", engine_lib); 94 95 dlerror(); 96 dlhandle = dlopen(engine_lib, RTLD_LAZY); 97 if (!dlhandle) { 98 td_vmsg(td, -1, dlerror(), "dlopen"); 99 return NULL; 100 } 101 102 /* 103 * Unlike the included modules, external engines should have a 104 * non-static ioengine structure that we can reference. 105 */ 106 ops = dlsym(dlhandle, "ioengine"); 107 if (!ops) { 108 td_vmsg(td, -1, dlerror(), "dlsym"); 109 dlclose(dlhandle); 110 return NULL; 111 } 112 113 ops->dlhandle = dlhandle; 114 return ops; 115} 116 117struct ioengine_ops *load_ioengine(struct thread_data *td, const char *name) 118{ 119 struct ioengine_ops *ops, *ret; 120 char engine[16]; 121 122 dprint(FD_IO, "load ioengine %s\n", name); 123 124 strncpy(engine, name, sizeof(engine) - 1); 125 126 /* 127 * linux libaio has alias names, so convert to what we want 128 */ 129 if (!strncmp(engine, "linuxaio", 8) || !strncmp(engine, "aio", 3)) 130 strcpy(engine, "libaio"); 131 132 ops = find_ioengine(engine); 133 if (!ops) 134 ops = dlopen_ioengine(td, name); 135 136 if (!ops) { 137 log_err("fio: engine %s not loadable\n", name); 138 return NULL; 139 } 140 141 /* 142 * Check that the required methods are there. 143 */ 144 if (check_engine_ops(ops)) 145 return NULL; 146 147 ret = malloc(sizeof(*ret)); 148 memcpy(ret, ops, sizeof(*ret)); 149 ret->data = NULL; 150 151 return ret; 152} 153 154void close_ioengine(struct thread_data *td) 155{ 156 dprint(FD_IO, "close ioengine %s\n", td->io_ops->name); 157 158 if (td->io_ops->cleanup) { 159 td->io_ops->cleanup(td); 160 td->io_ops->data = NULL; 161 } 162 163 if (td->io_ops->dlhandle) 164 dlclose(td->io_ops->dlhandle); 165 166 free(td->io_ops); 167 td->io_ops = NULL; 168} 169 170int td_io_prep(struct thread_data *td, struct io_u *io_u) 171{ 172 dprint_io_u(io_u, "prep"); 173 fio_ro_check(td, io_u); 174 175 lock_file(td, io_u->file, io_u->ddir); 176 177 if (td->io_ops->prep) { 178 int ret = td->io_ops->prep(td, io_u); 179 180 dprint(FD_IO, "->prep(%p)=%d\n", io_u, ret); 181 if (ret) 182 unlock_file(td, io_u->file); 183 return ret; 184 } 185 186 return 0; 187} 188 189int td_io_getevents(struct thread_data *td, unsigned int min, unsigned int max, 190 struct timespec *t) 191{ 192 int r = 0; 193 194 if (min > 0 && td->io_ops->commit) { 195 r = td->io_ops->commit(td); 196 if (r < 0) 197 goto out; 198 } 199 if (max > td->cur_depth) 200 max = td->cur_depth; 201 if (min > max) 202 max = min; 203 204 r = 0; 205 if (max && td->io_ops->getevents) 206 r = td->io_ops->getevents(td, min, max, t); 207out: 208 if (r >= 0) 209 io_u_mark_complete(td, r); 210 dprint(FD_IO, "getevents: %d\n", r); 211 return r; 212} 213 214int td_io_queue(struct thread_data *td, struct io_u *io_u) 215{ 216 int ret; 217 218 dprint_io_u(io_u, "queue"); 219 fio_ro_check(td, io_u); 220 221 assert((io_u->flags & IO_U_F_FLIGHT) == 0); 222 io_u->flags |= IO_U_F_FLIGHT; 223 224 assert(fio_file_open(io_u->file)); 225 226 io_u->error = 0; 227 io_u->resid = 0; 228 229 if (td->io_ops->flags & FIO_SYNCIO) { 230 if (fio_fill_issue_time(td)) 231 fio_gettime(&io_u->issue_time, NULL); 232 233 /* 234 * only used for iolog 235 */ 236 if (td->o.read_iolog_file) 237 memcpy(&td->last_issue, &io_u->issue_time, 238 sizeof(struct timeval)); 239 } 240 241 if (!ddir_sync(io_u->ddir)) 242 td->io_issues[io_u->ddir]++; 243 244 ret = td->io_ops->queue(td, io_u); 245 246 unlock_file(td, io_u->file); 247 248 /* 249 * Add warning for O_DIRECT so that users have an easier time 250 * spotting potentially bad alignment. If this triggers for the first 251 * IO, then it's likely an alignment problem or because the host fs 252 * does not support O_DIRECT 253 */ 254 if (io_u->error == EINVAL && td->io_issues[io_u->ddir] == 1 && 255 td->o.odirect) { 256 log_info("fio: first direct IO errored. File system may not " 257 "support direct IO, or iomem_align= is bad.\n"); 258 } 259 260 if (!td->io_ops->commit) { 261 io_u_mark_submit(td, 1); 262 io_u_mark_complete(td, 1); 263 } 264 265 if (ret == FIO_Q_COMPLETED) { 266 if (!ddir_sync(io_u->ddir)) { 267 io_u_mark_depth(td, 1); 268 td->ts.total_io_u[io_u->ddir]++; 269 } 270 } else if (ret == FIO_Q_QUEUED) { 271 int r; 272 273 if (!ddir_sync(io_u->ddir)) { 274 td->io_u_queued++; 275 td->ts.total_io_u[io_u->ddir]++; 276 } 277 278 if (td->io_u_queued >= td->o.iodepth_batch) { 279 r = td_io_commit(td); 280 if (r < 0) 281 return r; 282 } 283 } 284 285 if ((td->io_ops->flags & FIO_SYNCIO) == 0) { 286 if (fio_fill_issue_time(td)) 287 fio_gettime(&io_u->issue_time, NULL); 288 289 /* 290 * only used for iolog 291 */ 292 if (td->o.read_iolog_file) 293 memcpy(&td->last_issue, &io_u->issue_time, 294 sizeof(struct timeval)); 295 } 296 297 return ret; 298} 299 300int td_io_init(struct thread_data *td) 301{ 302 int ret = 0; 303 304 if (td->io_ops->init) { 305 ret = td->io_ops->init(td); 306 if (ret && td->o.iodepth > 1) { 307 log_err("fio: io engine init failed. Perhaps try" 308 " reducing io depth?\n"); 309 } 310 } 311 312 return ret; 313} 314 315int td_io_commit(struct thread_data *td) 316{ 317 dprint(FD_IO, "calling ->commit(), depth %d\n", td->cur_depth); 318 319 if (!td->cur_depth || !td->io_u_queued) 320 return 0; 321 322 io_u_mark_depth(td, td->io_u_queued); 323 td->io_u_queued = 0; 324 325 if (td->io_ops->commit) 326 return td->io_ops->commit(td); 327 328 return 0; 329} 330 331int td_io_open_file(struct thread_data *td, struct fio_file *f) 332{ 333 assert(!fio_file_open(f)); 334 assert(f->fd == -1); 335 336 if (td->io_ops->open_file(td, f)) { 337 if (td->error == EINVAL && td->o.odirect) 338 log_err("fio: destination does not support O_DIRECT\n"); 339 if (td->error == EMFILE) { 340 log_err("fio: try reducing/setting openfiles (failed" 341 " at %u of %u)\n", td->nr_open_files, 342 td->o.nr_files); 343 } 344 345 assert(f->fd == -1); 346 assert(!fio_file_open(f)); 347 return 1; 348 } 349 350 fio_file_reset(f); 351 fio_file_set_open(f); 352 fio_file_clear_closing(f); 353 disk_util_inc(f->du); 354 355 td->nr_open_files++; 356 get_file(f); 357 358 if (f->filetype == FIO_TYPE_PIPE) { 359 if (td_random(td)) { 360 log_err("fio: can't seek on pipes (no random io)\n"); 361 goto err; 362 } 363 } 364 365 if (td->io_ops->flags & FIO_DISKLESSIO) 366 goto done; 367 368 if (td->o.invalidate_cache && file_invalidate_cache(td, f)) 369 goto err; 370 371 if (td->o.fadvise_hint && 372 (f->filetype == FIO_TYPE_BD || f->filetype == FIO_TYPE_FILE)) { 373 int flags; 374 375 if (td_random(td)) 376 flags = POSIX_FADV_RANDOM; 377 else 378 flags = POSIX_FADV_SEQUENTIAL; 379 380 if (fadvise(f->fd, f->file_offset, f->io_size, flags) < 0) { 381 td_verror(td, errno, "fadvise"); 382 goto err; 383 } 384 } 385 386#ifdef FIO_OS_DIRECTIO 387 /* 388 * Some OS's have a distinct call to mark the file non-buffered, 389 * instead of using O_DIRECT (Solaris) 390 */ 391 if (td->o.odirect) { 392 int ret = fio_set_odirect(f->fd); 393 394 if (ret) { 395 td_verror(td, ret, "fio_set_odirect"); 396 goto err; 397 } 398 } 399#endif 400 401done: 402 log_file(td, f, FIO_LOG_OPEN_FILE); 403 return 0; 404err: 405 disk_util_dec(f->du); 406 if (td->io_ops->close_file) 407 td->io_ops->close_file(td, f); 408 return 1; 409} 410 411int td_io_close_file(struct thread_data *td, struct fio_file *f) 412{ 413 if (!fio_file_closing(f)) 414 log_file(td, f, FIO_LOG_CLOSE_FILE); 415 416 /* 417 * mark as closing, do real close when last io on it has completed 418 */ 419 fio_file_set_closing(f); 420 421 disk_util_dec(f->du); 422 unlock_file_all(td, f); 423 424 return put_file(td, f); 425} 426 427int td_io_get_file_size(struct thread_data *td, struct fio_file *f) 428{ 429 if (!td->io_ops->get_file_size) 430 return 0; 431 432 return td->io_ops->get_file_size(td, f); 433} 434 435static int do_sync_file_range(struct thread_data *td, struct fio_file *f) 436{ 437 off64_t offset, nbytes; 438 439 offset = f->first_write; 440 nbytes = f->last_write - f->first_write; 441 442 if (!nbytes) 443 return 0; 444 445 return sync_file_range(f->fd, offset, nbytes, td->o.sync_file_range); 446} 447 448int do_io_u_sync(struct thread_data *td, struct io_u *io_u) 449{ 450 int ret; 451 452 if (io_u->ddir == DDIR_SYNC) { 453 ret = fsync(io_u->file->fd); 454 } else if (io_u->ddir == DDIR_DATASYNC) { 455#ifdef FIO_HAVE_FDATASYNC 456 ret = fdatasync(io_u->file->fd); 457#else 458 ret = io_u->xfer_buflen; 459 io_u->error = EINVAL; 460#endif 461 } else if (io_u->ddir == DDIR_SYNC_FILE_RANGE) 462 ret = do_sync_file_range(td, io_u->file); 463 else { 464 ret = io_u->xfer_buflen; 465 io_u->error = EINVAL; 466 } 467 468 if (ret < 0) 469 io_u->error = errno; 470 471 return ret; 472} 473