12866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe/* 2da751ca9665bcdeca56d2eec5b629a0953c07662Jens Axboe * libaio engine 3da751ca9665bcdeca56d2eec5b629a0953c07662Jens Axboe * 4da751ca9665bcdeca56d2eec5b629a0953c07662Jens Axboe * IO engine using the Linux native aio interface. 52866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe * 62866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe */ 72866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <stdio.h> 82866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <stdlib.h> 92866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <unistd.h> 102866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <errno.h> 112866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <assert.h> 1267bf982340d95ca98098ea050b54b4c7adb116c0Jens Axboe#include <libaio.h> 135f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe 145f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe#include "../fio.h" 152866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 1617876dcdbc2a256c7de615eea03d40d09134d98dJens Axboestatic int fio_libaio_commit(struct thread_data *td); 1717876dcdbc2a256c7de615eea03d40d09134d98dJens Axboe 182866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestruct libaio_data { 192866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe io_context_t aio_ctx; 202866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct io_event *aio_events; 21755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe struct iocb **iocbs; 227e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe struct io_u **io_us; 23241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe 24241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe /* 25241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe * Basic ring buffer. 'head' is incremented in _queue(), and 26241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe * 'tail' is incremented in _commit(). We keep 'queued' so 27241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe * that we know if the ring is full or empty, when 28241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe * 'head' == 'tail'. 'entries' is the ring size, and 29241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe * 'is_pow2' is just an optimization to use AND instead of 30241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe * modulus to get the remainder on ring increment. 31241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe */ 32241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe int is_pow2; 33241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe unsigned int entries; 34241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe unsigned int queued; 35241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe unsigned int head; 36241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe unsigned int tail; 372866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe}; 382866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 39de890a1e48d40238dac69f302708dde8719de240Steven Langstruct libaio_options { 406a605302a1f0b63075cfbe7144122bcdc61d61ccJens Axboe void *pad; 41de890a1e48d40238dac69f302708dde8719de240Steven Lang unsigned int userspace_reap; 42de890a1e48d40238dac69f302708dde8719de240Steven Lang}; 43de890a1e48d40238dac69f302708dde8719de240Steven Lang 44de890a1e48d40238dac69f302708dde8719de240Steven Langstatic struct fio_option options[] = { 45de890a1e48d40238dac69f302708dde8719de240Steven Lang { 46de890a1e48d40238dac69f302708dde8719de240Steven Lang .name = "userspace_reap", 47e8b0e958cd219cabb1154e2b06036863a7d6dbd7Jens Axboe .lname = "Libaio userspace reaping", 48de890a1e48d40238dac69f302708dde8719de240Steven Lang .type = FIO_OPT_STR_SET, 49de890a1e48d40238dac69f302708dde8719de240Steven Lang .off1 = offsetof(struct libaio_options, userspace_reap), 50de890a1e48d40238dac69f302708dde8719de240Steven Lang .help = "Use alternative user-space reap implementation", 51e90a0adf85f75a65c02b22ba3766dff0d9d82ae6Jens Axboe .category = FIO_OPT_C_ENGINE, 525a3cd5f3e9b4a38c961279cb8616ac1803735139Jens Axboe .group = FIO_OPT_G_LIBAIO, 53de890a1e48d40238dac69f302708dde8719de240Steven Lang }, 54de890a1e48d40238dac69f302708dde8719de240Steven Lang { 55de890a1e48d40238dac69f302708dde8719de240Steven Lang .name = NULL, 56de890a1e48d40238dac69f302708dde8719de240Steven Lang }, 57de890a1e48d40238dac69f302708dde8719de240Steven Lang}; 58de890a1e48d40238dac69f302708dde8719de240Steven Lang 59241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboestatic inline void ring_inc(struct libaio_data *ld, unsigned int *val, 60241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe unsigned int add) 61241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe{ 62241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe if (ld->is_pow2) 63241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe *val = (*val + add) & (ld->entries - 1); 64241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe else 65241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe *val = (*val + add) % ld->entries; 66241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe} 67241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe 687a16dd027f83d7ff8ff0e72bb6bb12fae046fcbfJens Axboestatic int fio_libaio_prep(struct thread_data fio_unused *td, struct io_u *io_u) 692866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 7053cdc6864f7471b28cc9b40a5314ab43e5b1cb5eJens Axboe struct fio_file *f = io_u->file; 7153cdc6864f7471b28cc9b40a5314ab43e5b1cb5eJens Axboe 722866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe if (io_u->ddir == DDIR_READ) 73cec6b55da1c282b5b91ad346c7804171fccf151eJens Axboe io_prep_pread(&io_u->iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset); 7487dc1ab1b4df7b977f60e3d43533a896e2ee665bJens Axboe else if (io_u->ddir == DDIR_WRITE) 75cec6b55da1c282b5b91ad346c7804171fccf151eJens Axboe io_prep_pwrite(&io_u->iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset); 765f9099ea8adf423d0db01274ef0d7e65629c0e1cJens Axboe else if (ddir_sync(io_u->ddir)) 7787dc1ab1b4df7b977f60e3d43533a896e2ee665bJens Axboe io_prep_fsync(&io_u->iocb, f->fd); 782866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 792866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe return 0; 802866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 812866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 822866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic struct io_u *fio_libaio_event(struct thread_data *td, int event) 832866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 842866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = td->io_ops->data; 85f423479dad8916c758792308863dcd833a1a36c8Jens Axboe struct io_event *ev; 86f423479dad8916c758792308863dcd833a1a36c8Jens Axboe struct io_u *io_u; 872866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 88f423479dad8916c758792308863dcd833a1a36c8Jens Axboe ev = ld->aio_events + event; 890cc6ee59fd3caf08e934438241afa4982a15642cJens Axboe io_u = container_of(ev->obj, struct io_u, iocb); 90f423479dad8916c758792308863dcd833a1a36c8Jens Axboe 91f423479dad8916c758792308863dcd833a1a36c8Jens Axboe if (ev->res != io_u->xfer_buflen) { 92f423479dad8916c758792308863dcd833a1a36c8Jens Axboe if (ev->res > io_u->xfer_buflen) 93f423479dad8916c758792308863dcd833a1a36c8Jens Axboe io_u->error = -ev->res; 94f423479dad8916c758792308863dcd833a1a36c8Jens Axboe else 95f423479dad8916c758792308863dcd833a1a36c8Jens Axboe io_u->resid = io_u->xfer_buflen - ev->res; 96f423479dad8916c758792308863dcd833a1a36c8Jens Axboe } else 97f423479dad8916c758792308863dcd833a1a36c8Jens Axboe io_u->error = 0; 98f423479dad8916c758792308863dcd833a1a36c8Jens Axboe 99f423479dad8916c758792308863dcd833a1a36c8Jens Axboe return io_u; 1002866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 1012866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 102675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenbergstruct aio_ring { 103675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned id; /** kernel internal index number */ 104675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned nr; /** number of io_events */ 105675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned head; 106675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned tail; 107c44b1ff54402c589e8f07436ec56efb4f8b1ac23Jens Axboe 108675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned magic; 109675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned compat_features; 110675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned incompat_features; 111675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned header_length; /** size of aio_ring */ 112675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 113675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg struct io_event events[0]; 114675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg}; 115675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 116675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg#define AIO_RING_MAGIC 0xa10a10a1 117675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 118675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenbergstatic int user_io_getevents(io_context_t aio_ctx, unsigned int max, 119c44b1ff54402c589e8f07436ec56efb4f8b1ac23Jens Axboe struct io_event *events) 120675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg{ 121675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg long i = 0; 122675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned head; 123c44b1ff54402c589e8f07436ec56efb4f8b1ac23Jens Axboe struct aio_ring *ring = (struct aio_ring*) aio_ctx; 124675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 125675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg while (i < max) { 126675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg head = ring->head; 127675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 128675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg if (head == ring->tail) { 129675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg /* There are no more completions */ 130675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg break; 131675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg } else { 132675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg /* There is another completion to reap */ 133675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg events[i] = ring->events[head]; 134675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg read_barrier(); 135c44b1ff54402c589e8f07436ec56efb4f8b1ac23Jens Axboe ring->head = (head + 1) % ring->nr; 136675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg i++; 137675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg } 138675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg } 139675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 140675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg return i; 141675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg} 142675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 143e7d2e61694c62b90a2fb84c012b4edcc1973d72cJens Axboestatic int fio_libaio_getevents(struct thread_data *td, unsigned int min, 1440cbbc3919c80d969926f499cb6045be3d5cd0305Jens Axboe unsigned int max, const struct timespec *t) 1452866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 1462866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = td->io_ops->data; 147de890a1e48d40238dac69f302708dde8719de240Steven Lang struct libaio_options *o = td->eo; 1480b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg unsigned actual_min = td->o.iodepth_batch_complete == 0 ? 0 : min; 1490cbbc3919c80d969926f499cb6045be3d5cd0305Jens Axboe struct timespec __lt, *lt = NULL; 1500b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg int r, events = 0; 1512866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 1520cbbc3919c80d969926f499cb6045be3d5cd0305Jens Axboe if (t) { 1530cbbc3919c80d969926f499cb6045be3d5cd0305Jens Axboe __lt = *t; 1540cbbc3919c80d969926f499cb6045be3d5cd0305Jens Axboe lt = &__lt; 1550cbbc3919c80d969926f499cb6045be3d5cd0305Jens Axboe } 1560cbbc3919c80d969926f499cb6045be3d5cd0305Jens Axboe 1572866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe do { 158de890a1e48d40238dac69f302708dde8719de240Steven Lang if (o->userspace_reap == 1 159675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg && actual_min == 0 160675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg && ((struct aio_ring *)(ld->aio_ctx))->magic 161675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg == AIO_RING_MAGIC) { 162675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg r = user_io_getevents(ld->aio_ctx, max, 163675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg ld->aio_events + events); 164675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg } else { 165675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg r = io_getevents(ld->aio_ctx, actual_min, 1660cbbc3919c80d969926f499cb6045be3d5cd0305Jens Axboe max, ld->aio_events + events, lt); 167675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg } 16826166ef7c7090fb8791d17fcd635cbb0a93c694cAndrey Kuzmin if (r > 0) 1690b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg events += r; 17026166ef7c7090fb8791d17fcd635cbb0a93c694cAndrey Kuzmin else if ((min && r == 0) || r == -EAGAIN) { 17117876dcdbc2a256c7de615eea03d40d09134d98dJens Axboe fio_libaio_commit(td); 1722866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe usleep(100); 17317876dcdbc2a256c7de615eea03d40d09134d98dJens Axboe } else if (r != -EINTR) 1740c195ae51db8a15e20d117afb88423118baafc11Jens Axboe break; 1750b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg } while (events < min); 1760b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg 1770b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg return r < 0 ? r : events; 1782866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 1792866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 1802866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic int fio_libaio_queue(struct thread_data *td, struct io_u *io_u) 1812866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 1822866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = td->io_ops->data; 1832866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 1847101d9c24abec4be58a086d85d6d92ec6e6492e9Jens Axboe fio_ro_check(td, io_u); 1857101d9c24abec4be58a086d85d6d92ec6e6492e9Jens Axboe 186241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe if (ld->queued == td->o.iodepth) 187755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe return FIO_Q_BUSY; 188755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 189755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe /* 190755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe * fsync is tricky, since it can fail and we need to do it 191755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe * serialized with other io. the reason is that linux doesn't 192755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe * support aio fsync yet. So return busy for the case where we 193755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe * have pending io, to let fio complete those first. 194755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe */ 195f011531e61ae750cdf82074e0dea1379b07fa239Jens Axboe if (ddir_sync(io_u->ddir)) { 196241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe if (ld->queued) 197755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe return FIO_Q_BUSY; 1985f9099ea8adf423d0db01274ef0d7e65629c0e1cJens Axboe 199f011531e61ae750cdf82074e0dea1379b07fa239Jens Axboe do_io_u_sync(td, io_u); 2005f9099ea8adf423d0db01274ef0d7e65629c0e1cJens Axboe return FIO_Q_COMPLETED; 201755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe } 202755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 203a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe if (io_u->ddir == DDIR_TRIM) { 204241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe if (ld->queued) 205a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe return FIO_Q_BUSY; 206a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe 207a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe do_io_u_trim(td, io_u); 208a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe return FIO_Q_COMPLETED; 209a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe } 210a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe 211241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ld->iocbs[ld->head] = &io_u->iocb; 212241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ld->io_us[ld->head] = io_u; 213241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ring_inc(ld, &ld->head, 1); 214241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ld->queued++; 215755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe return FIO_Q_QUEUED; 216755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe} 217755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 2187e77dd026d85253936aef432ba8f3e89b96b805cJens Axboestatic void fio_libaio_queued(struct thread_data *td, struct io_u **io_us, 2197e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe unsigned int nr) 2207e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe{ 2217e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe struct timeval now; 2227e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe unsigned int i; 2237e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe 22412d9d841526ad75a67bb43a90edeefd05f85f11eJens Axboe if (!fio_fill_issue_time(td)) 22512d9d841526ad75a67bb43a90edeefd05f85f11eJens Axboe return; 22612d9d841526ad75a67bb43a90edeefd05f85f11eJens Axboe 2277e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe fio_gettime(&now, NULL); 2287e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe 2297e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe for (i = 0; i < nr; i++) { 2307e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe struct io_u *io_u = io_us[i]; 2317e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe 2327e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe memcpy(&io_u->issue_time, &now, sizeof(now)); 2337e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe io_u_queued(td, io_u); 2347e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe } 2357e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe} 2367e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe 237755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboestatic int fio_libaio_commit(struct thread_data *td) 238755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe{ 239755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe struct libaio_data *ld = td->io_ops->data; 240755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe struct iocb **iocbs; 2417e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe struct io_u **io_us; 242841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe struct timeval tv; 243841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe int ret, wait_start = 0; 244755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 245241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe if (!ld->queued) 246755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe return 0; 247755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 2482866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe do { 249241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe long nr = ld->queued; 250241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe 251241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe nr = min((unsigned int) nr, ld->entries - ld->tail); 252241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe io_us = ld->io_us + ld->tail; 253241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe iocbs = ld->iocbs + ld->tail; 254241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe 255241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ret = io_submit(ld->aio_ctx, nr, iocbs); 2565e00c2c4b7e2d5c1e535dbe10ee4b687f9f4b72bJens Axboe if (ret > 0) { 2577e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe fio_libaio_queued(td, io_us, ret); 258838bc709279964bdcc64070d4eb2777a0f79bcbbJens Axboe io_u_mark_submit(td, ret); 259241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe 260241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ld->queued -= ret; 261241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ring_inc(ld, &ld->tail, ret); 2625e00c2c4b7e2d5c1e535dbe10ee4b687f9f4b72bJens Axboe ret = 0; 2638bc7692033670330dffe0d0e47c4a3bf5bdebd49Jens Axboe wait_start = 0; 264241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe } else if (ret == -EINTR || !ret) { 265838bc709279964bdcc64070d4eb2777a0f79bcbbJens Axboe if (!ret) 266838bc709279964bdcc64070d4eb2777a0f79bcbbJens Axboe io_u_mark_submit(td, ret); 2678bc7692033670330dffe0d0e47c4a3bf5bdebd49Jens Axboe wait_start = 0; 2682866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe continue; 269241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe } else if (ret == -EAGAIN) { 270241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe /* 271241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe * If we get EAGAIN, we should break out without 272241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe * error and let the upper layer reap some 273841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe * events for us. If we have no queued IO, we 274841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe * must loop here. If we loop for more than 30s, 275841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe * just error out, something must be buggy in the 276841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe * IO path. 277241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe */ 278841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe if (ld->queued) { 279841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe ret = 0; 280841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe break; 281841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe } 282841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe if (!wait_start) { 283841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe fio_gettime(&tv, NULL); 2849b093d8fd80ef001dfa777f1edc5a7c49a37b943Jens Axboe wait_start = 1; 285841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe } else if (mtime_since_now(&tv) > 30000) { 286841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe log_err("fio: aio appears to be stalled, giving up\n"); 287841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe break; 288841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe } 289841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe usleep(1); 290841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe continue; 2910c195ae51db8a15e20d117afb88423118baafc11Jens Axboe } else if (ret == -ENOMEM) { 2920c195ae51db8a15e20d117afb88423118baafc11Jens Axboe /* 2930c195ae51db8a15e20d117afb88423118baafc11Jens Axboe * If we get -ENOMEM, reap events if we can. If 2940c195ae51db8a15e20d117afb88423118baafc11Jens Axboe * we cannot, treat it as a fatal event since there's 2950c195ae51db8a15e20d117afb88423118baafc11Jens Axboe * nothing we can do about it. 2960c195ae51db8a15e20d117afb88423118baafc11Jens Axboe */ 2970c195ae51db8a15e20d117afb88423118baafc11Jens Axboe if (ld->queued) 2980c195ae51db8a15e20d117afb88423118baafc11Jens Axboe ret = 0; 2990c195ae51db8a15e20d117afb88423118baafc11Jens Axboe break; 300838bc709279964bdcc64070d4eb2777a0f79bcbbJens Axboe } else 3012866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe break; 30244bbe1093e9ed07e41edd061c4171896a900ad9fJens Axboe } while (ld->queued); 3032866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 30436167d82e5f49dee91c6d2cd426068edee90e36fJens Axboe return ret; 3052866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 3062866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 3072866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic int fio_libaio_cancel(struct thread_data *td, struct io_u *io_u) 3082866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 3092866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = td->io_ops->data; 3102866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 3112866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe return io_cancel(ld->aio_ctx, &io_u->iocb, ld->aio_events); 3122866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 3132866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 3142866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic void fio_libaio_cleanup(struct thread_data *td) 3152866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 3162866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = td->io_ops->data; 3172866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 3182866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe if (ld) { 3192866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe io_destroy(ld->aio_ctx); 3207e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe free(ld->aio_events); 3217e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe free(ld->iocbs); 3227e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe free(ld->io_us); 3232866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe free(ld); 3242866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe } 3252866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 3262866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 3272866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic int fio_libaio_init(struct thread_data *td) 3282866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 329c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe struct libaio_options *o = td->eo; 330241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe struct libaio_data *ld; 331c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe int err = 0; 3322866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 333241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ld = calloc(1, sizeof(*ld)); 334c1db2dced4750f2f506cf4df232dcc75ae05b950Jens Axboe 335c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe /* 336c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe * First try passing in 0 for queue depth, since we don't 337c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe * care about the user ring. If that fails, the kernel is too old 338c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe * and we need the right depth. 339c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe */ 340c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe if (!o->userspace_reap) 341c224ec052b3a5be4fba8da777dd7ccbee2fb34c1Jens Axboe err = io_queue_init(INT_MAX, &ld->aio_ctx); 342c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe if (o->userspace_reap || err == -EINVAL) 343c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe err = io_queue_init(td->o.iodepth, &ld->aio_ctx); 344c1db2dced4750f2f506cf4df232dcc75ae05b950Jens Axboe if (err) { 345c1db2dced4750f2f506cf4df232dcc75ae05b950Jens Axboe td_verror(td, -err, "io_queue_init"); 34675de55acafe8663389449517621da67347354d62Jens Axboe log_err("fio: check /proc/sys/fs/aio-max-nr\n"); 347cb781c758e5df7ff4c6f655c1e4f9df3043a4be9Jens Axboe free(ld); 3482866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe return 1; 3492866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe } 3502866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 351241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ld->entries = td->o.iodepth; 352241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ld->is_pow2 = is_power_of_2(ld->entries); 353241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ld->aio_events = calloc(ld->entries, sizeof(struct io_event)); 354241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ld->iocbs = calloc(ld->entries, sizeof(struct iocb *)); 355241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ld->io_us = calloc(ld->entries, sizeof(struct io_u *)); 356755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 3572866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe td->io_ops->data = ld; 3582866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe return 0; 3592866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 3602866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 3615f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboestatic struct ioengine_ops ioengine = { 362de890a1e48d40238dac69f302708dde8719de240Steven Lang .name = "libaio", 363de890a1e48d40238dac69f302708dde8719de240Steven Lang .version = FIO_IOOPS_VERSION, 364de890a1e48d40238dac69f302708dde8719de240Steven Lang .init = fio_libaio_init, 365de890a1e48d40238dac69f302708dde8719de240Steven Lang .prep = fio_libaio_prep, 366de890a1e48d40238dac69f302708dde8719de240Steven Lang .queue = fio_libaio_queue, 367de890a1e48d40238dac69f302708dde8719de240Steven Lang .commit = fio_libaio_commit, 368de890a1e48d40238dac69f302708dde8719de240Steven Lang .cancel = fio_libaio_cancel, 369de890a1e48d40238dac69f302708dde8719de240Steven Lang .getevents = fio_libaio_getevents, 370de890a1e48d40238dac69f302708dde8719de240Steven Lang .event = fio_libaio_event, 371de890a1e48d40238dac69f302708dde8719de240Steven Lang .cleanup = fio_libaio_cleanup, 372de890a1e48d40238dac69f302708dde8719de240Steven Lang .open_file = generic_open_file, 373de890a1e48d40238dac69f302708dde8719de240Steven Lang .close_file = generic_close_file, 374de890a1e48d40238dac69f302708dde8719de240Steven Lang .get_file_size = generic_get_file_size, 375de890a1e48d40238dac69f302708dde8719de240Steven Lang .options = options, 376de890a1e48d40238dac69f302708dde8719de240Steven Lang .option_struct_size = sizeof(struct libaio_options), 3772866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe}; 37834cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe 3795f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboestatic void fio_init fio_libaio_register(void) 3805f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe{ 3815f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe register_ioengine(&ioengine); 3825f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe} 3835f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe 3845f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboestatic void fio_exit fio_libaio_unregister(void) 3855f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe{ 3865f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe unregister_ioengine(&ioengine); 3875f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe} 388