libaio.c revision 44bbe1093e9ed07e41edd061c4171896a900ad9f
12866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe/* 2da751ca9665bcdeca56d2eec5b629a0953c07662Jens Axboe * libaio engine 3da751ca9665bcdeca56d2eec5b629a0953c07662Jens Axboe * 4da751ca9665bcdeca56d2eec5b629a0953c07662Jens Axboe * IO engine using the Linux native aio interface. 52866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe * 62866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe */ 72866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <stdio.h> 82866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <stdlib.h> 92866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <unistd.h> 102866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <errno.h> 112866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <assert.h> 1267bf982340d95ca98098ea050b54b4c7adb116c0Jens Axboe#include <libaio.h> 135f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe 145f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe#include "../fio.h" 152866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 162866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestruct libaio_data { 172866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe io_context_t aio_ctx; 182866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct io_event *aio_events; 19755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe struct iocb **iocbs; 207e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe struct io_u **io_us; 21241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe 22241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe /* 23241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe * Basic ring buffer. 'head' is incremented in _queue(), and 24241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe * 'tail' is incremented in _commit(). We keep 'queued' so 25241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe * that we know if the ring is full or empty, when 26241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe * 'head' == 'tail'. 'entries' is the ring size, and 27241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe * 'is_pow2' is just an optimization to use AND instead of 28241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe * modulus to get the remainder on ring increment. 29241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe */ 30241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe int is_pow2; 31241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe unsigned int entries; 32241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe unsigned int queued; 33241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe unsigned int head; 34241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe unsigned int tail; 352866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe}; 362866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 37de890a1e48d40238dac69f302708dde8719de240Steven Langstruct libaio_options { 38de890a1e48d40238dac69f302708dde8719de240Steven Lang struct thread_data *td; 39de890a1e48d40238dac69f302708dde8719de240Steven Lang unsigned int userspace_reap; 40de890a1e48d40238dac69f302708dde8719de240Steven Lang}; 41de890a1e48d40238dac69f302708dde8719de240Steven Lang 42de890a1e48d40238dac69f302708dde8719de240Steven Langstatic struct fio_option options[] = { 43de890a1e48d40238dac69f302708dde8719de240Steven Lang { 44de890a1e48d40238dac69f302708dde8719de240Steven Lang .name = "userspace_reap", 45e8b0e958cd219cabb1154e2b06036863a7d6dbd7Jens Axboe .lname = "Libaio userspace reaping", 46de890a1e48d40238dac69f302708dde8719de240Steven Lang .type = FIO_OPT_STR_SET, 47de890a1e48d40238dac69f302708dde8719de240Steven Lang .off1 = offsetof(struct libaio_options, userspace_reap), 48de890a1e48d40238dac69f302708dde8719de240Steven Lang .help = "Use alternative user-space reap implementation", 49e90a0adf85f75a65c02b22ba3766dff0d9d82ae6Jens Axboe .category = FIO_OPT_C_ENGINE, 505a3cd5f3e9b4a38c961279cb8616ac1803735139Jens Axboe .group = FIO_OPT_G_LIBAIO, 51de890a1e48d40238dac69f302708dde8719de240Steven Lang }, 52de890a1e48d40238dac69f302708dde8719de240Steven Lang { 53de890a1e48d40238dac69f302708dde8719de240Steven Lang .name = NULL, 54de890a1e48d40238dac69f302708dde8719de240Steven Lang }, 55de890a1e48d40238dac69f302708dde8719de240Steven Lang}; 56de890a1e48d40238dac69f302708dde8719de240Steven Lang 57241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboestatic inline void ring_inc(struct libaio_data *ld, unsigned int *val, 58241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe unsigned int add) 59241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe{ 60241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe if (ld->is_pow2) 61241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe *val = (*val + add) & (ld->entries - 1); 62241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe else 63241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe *val = (*val + add) % ld->entries; 64241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe} 65241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe 667a16dd027f83d7ff8ff0e72bb6bb12fae046fcbfJens Axboestatic int fio_libaio_prep(struct thread_data fio_unused *td, struct io_u *io_u) 672866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 6853cdc6864f7471b28cc9b40a5314ab43e5b1cb5eJens Axboe struct fio_file *f = io_u->file; 6953cdc6864f7471b28cc9b40a5314ab43e5b1cb5eJens Axboe 702866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe if (io_u->ddir == DDIR_READ) 71cec6b55da1c282b5b91ad346c7804171fccf151eJens Axboe io_prep_pread(&io_u->iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset); 7287dc1ab1b4df7b977f60e3d43533a896e2ee665bJens Axboe else if (io_u->ddir == DDIR_WRITE) 73cec6b55da1c282b5b91ad346c7804171fccf151eJens Axboe io_prep_pwrite(&io_u->iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset); 745f9099ea8adf423d0db01274ef0d7e65629c0e1cJens Axboe else if (ddir_sync(io_u->ddir)) 7587dc1ab1b4df7b977f60e3d43533a896e2ee665bJens Axboe io_prep_fsync(&io_u->iocb, f->fd); 762866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 772866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe return 0; 782866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 792866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 802866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic struct io_u *fio_libaio_event(struct thread_data *td, int event) 812866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 822866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = td->io_ops->data; 83f423479dad8916c758792308863dcd833a1a36c8Jens Axboe struct io_event *ev; 84f423479dad8916c758792308863dcd833a1a36c8Jens Axboe struct io_u *io_u; 852866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 86f423479dad8916c758792308863dcd833a1a36c8Jens Axboe ev = ld->aio_events + event; 870cc6ee59fd3caf08e934438241afa4982a15642cJens Axboe io_u = container_of(ev->obj, struct io_u, iocb); 88f423479dad8916c758792308863dcd833a1a36c8Jens Axboe 89f423479dad8916c758792308863dcd833a1a36c8Jens Axboe if (ev->res != io_u->xfer_buflen) { 90f423479dad8916c758792308863dcd833a1a36c8Jens Axboe if (ev->res > io_u->xfer_buflen) 91f423479dad8916c758792308863dcd833a1a36c8Jens Axboe io_u->error = -ev->res; 92f423479dad8916c758792308863dcd833a1a36c8Jens Axboe else 93f423479dad8916c758792308863dcd833a1a36c8Jens Axboe io_u->resid = io_u->xfer_buflen - ev->res; 94f423479dad8916c758792308863dcd833a1a36c8Jens Axboe } else 95f423479dad8916c758792308863dcd833a1a36c8Jens Axboe io_u->error = 0; 96f423479dad8916c758792308863dcd833a1a36c8Jens Axboe 97f423479dad8916c758792308863dcd833a1a36c8Jens Axboe return io_u; 982866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 992866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 100675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenbergstruct aio_ring { 101675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned id; /** kernel internal index number */ 102675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned nr; /** number of io_events */ 103675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned head; 104675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned tail; 105c44b1ff54402c589e8f07436ec56efb4f8b1ac23Jens Axboe 106675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned magic; 107675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned compat_features; 108675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned incompat_features; 109675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned header_length; /** size of aio_ring */ 110675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 111675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg struct io_event events[0]; 112675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg}; 113675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 114675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg#define AIO_RING_MAGIC 0xa10a10a1 115675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 116675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenbergstatic int user_io_getevents(io_context_t aio_ctx, unsigned int max, 117c44b1ff54402c589e8f07436ec56efb4f8b1ac23Jens Axboe struct io_event *events) 118675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg{ 119675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg long i = 0; 120675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned head; 121c44b1ff54402c589e8f07436ec56efb4f8b1ac23Jens Axboe struct aio_ring *ring = (struct aio_ring*) aio_ctx; 122675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 123675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg while (i < max) { 124675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg head = ring->head; 125675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 126675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg if (head == ring->tail) { 127675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg /* There are no more completions */ 128675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg break; 129675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg } else { 130675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg /* There is another completion to reap */ 131675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg events[i] = ring->events[head]; 132675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg read_barrier(); 133c44b1ff54402c589e8f07436ec56efb4f8b1ac23Jens Axboe ring->head = (head + 1) % ring->nr; 134675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg i++; 135675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg } 136675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg } 137675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 138675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg return i; 139675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg} 140675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 141e7d2e61694c62b90a2fb84c012b4edcc1973d72cJens Axboestatic int fio_libaio_getevents(struct thread_data *td, unsigned int min, 1420cbbc3919c80d969926f499cb6045be3d5cd0305Jens Axboe unsigned int max, const struct timespec *t) 1432866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 1442866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = td->io_ops->data; 145de890a1e48d40238dac69f302708dde8719de240Steven Lang struct libaio_options *o = td->eo; 1460b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg unsigned actual_min = td->o.iodepth_batch_complete == 0 ? 0 : min; 1470cbbc3919c80d969926f499cb6045be3d5cd0305Jens Axboe struct timespec __lt, *lt = NULL; 1480b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg int r, events = 0; 1492866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 1500cbbc3919c80d969926f499cb6045be3d5cd0305Jens Axboe if (t) { 1510cbbc3919c80d969926f499cb6045be3d5cd0305Jens Axboe __lt = *t; 1520cbbc3919c80d969926f499cb6045be3d5cd0305Jens Axboe lt = &__lt; 1530cbbc3919c80d969926f499cb6045be3d5cd0305Jens Axboe } 1540cbbc3919c80d969926f499cb6045be3d5cd0305Jens Axboe 1552866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe do { 156de890a1e48d40238dac69f302708dde8719de240Steven Lang if (o->userspace_reap == 1 157675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg && actual_min == 0 158675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg && ((struct aio_ring *)(ld->aio_ctx))->magic 159675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg == AIO_RING_MAGIC) { 160675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg r = user_io_getevents(ld->aio_ctx, max, 161675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg ld->aio_events + events); 162675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg } else { 163675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg r = io_getevents(ld->aio_ctx, actual_min, 1640cbbc3919c80d969926f499cb6045be3d5cd0305Jens Axboe max, ld->aio_events + events, lt); 165675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg } 1660b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg if (r >= 0) 1670b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg events += r; 1680b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg else if (r == -EAGAIN) 1692866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe usleep(100); 1700b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg } while (events < min); 1710b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg 1720b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg return r < 0 ? r : events; 1732866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 1742866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 1752866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic int fio_libaio_queue(struct thread_data *td, struct io_u *io_u) 1762866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 1772866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = td->io_ops->data; 1782866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 1797101d9c24abec4be58a086d85d6d92ec6e6492e9Jens Axboe fio_ro_check(td, io_u); 1807101d9c24abec4be58a086d85d6d92ec6e6492e9Jens Axboe 181241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe if (ld->queued == td->o.iodepth) 182755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe return FIO_Q_BUSY; 183755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 184755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe /* 185755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe * fsync is tricky, since it can fail and we need to do it 186755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe * serialized with other io. the reason is that linux doesn't 187755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe * support aio fsync yet. So return busy for the case where we 188755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe * have pending io, to let fio complete those first. 189755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe */ 190f011531e61ae750cdf82074e0dea1379b07fa239Jens Axboe if (ddir_sync(io_u->ddir)) { 191241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe if (ld->queued) 192755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe return FIO_Q_BUSY; 1935f9099ea8adf423d0db01274ef0d7e65629c0e1cJens Axboe 194f011531e61ae750cdf82074e0dea1379b07fa239Jens Axboe do_io_u_sync(td, io_u); 1955f9099ea8adf423d0db01274ef0d7e65629c0e1cJens Axboe return FIO_Q_COMPLETED; 196755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe } 197755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 198a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe if (io_u->ddir == DDIR_TRIM) { 199241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe if (ld->queued) 200a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe return FIO_Q_BUSY; 201a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe 202a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe do_io_u_trim(td, io_u); 203a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe return FIO_Q_COMPLETED; 204a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe } 205a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe 206241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ld->iocbs[ld->head] = &io_u->iocb; 207241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ld->io_us[ld->head] = io_u; 208241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ring_inc(ld, &ld->head, 1); 209241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ld->queued++; 210755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe return FIO_Q_QUEUED; 211755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe} 212755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 2137e77dd026d85253936aef432ba8f3e89b96b805cJens Axboestatic void fio_libaio_queued(struct thread_data *td, struct io_u **io_us, 2147e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe unsigned int nr) 2157e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe{ 2167e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe struct timeval now; 2177e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe unsigned int i; 2187e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe 21912d9d841526ad75a67bb43a90edeefd05f85f11eJens Axboe if (!fio_fill_issue_time(td)) 22012d9d841526ad75a67bb43a90edeefd05f85f11eJens Axboe return; 22112d9d841526ad75a67bb43a90edeefd05f85f11eJens Axboe 2227e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe fio_gettime(&now, NULL); 2237e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe 2247e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe for (i = 0; i < nr; i++) { 2257e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe struct io_u *io_u = io_us[i]; 2267e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe 2277e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe memcpy(&io_u->issue_time, &now, sizeof(now)); 2287e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe io_u_queued(td, io_u); 2297e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe } 2307e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe} 2317e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe 232755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboestatic int fio_libaio_commit(struct thread_data *td) 233755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe{ 234755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe struct libaio_data *ld = td->io_ops->data; 235755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe struct iocb **iocbs; 2367e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe struct io_u **io_us; 237841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe struct timeval tv; 238841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe int ret, wait_start = 0; 239755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 240241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe if (!ld->queued) 241755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe return 0; 242755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 2432866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe do { 244241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe long nr = ld->queued; 245241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe 246241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe nr = min((unsigned int) nr, ld->entries - ld->tail); 247241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe io_us = ld->io_us + ld->tail; 248241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe iocbs = ld->iocbs + ld->tail; 249241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe 250241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ret = io_submit(ld->aio_ctx, nr, iocbs); 2515e00c2c4b7e2d5c1e535dbe10ee4b687f9f4b72bJens Axboe if (ret > 0) { 2527e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe fio_libaio_queued(td, io_us, ret); 253838bc709279964bdcc64070d4eb2777a0f79bcbbJens Axboe io_u_mark_submit(td, ret); 254241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe 255241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ld->queued -= ret; 256241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ring_inc(ld, &ld->tail, ret); 2575e00c2c4b7e2d5c1e535dbe10ee4b687f9f4b72bJens Axboe ret = 0; 258241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe } else if (ret == -EINTR || !ret) { 259838bc709279964bdcc64070d4eb2777a0f79bcbbJens Axboe if (!ret) 260838bc709279964bdcc64070d4eb2777a0f79bcbbJens Axboe io_u_mark_submit(td, ret); 2612866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe continue; 262241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe } else if (ret == -EAGAIN) { 263241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe /* 264241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe * If we get EAGAIN, we should break out without 265241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe * error and let the upper layer reap some 266841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe * events for us. If we have no queued IO, we 267841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe * must loop here. If we loop for more than 30s, 268841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe * just error out, something must be buggy in the 269841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe * IO path. 270241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe */ 271841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe if (ld->queued) { 272841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe ret = 0; 273841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe break; 274841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe } 275841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe if (!wait_start) { 276841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe fio_gettime(&tv, NULL); 277841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe wait_start = 0; 278841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe } else if (mtime_since_now(&tv) > 30000) { 279841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe log_err("fio: aio appears to be stalled, giving up\n"); 280841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe break; 281841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe } 282841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe usleep(1); 283841ffe5ce7eab44ab934af559933222bf13a2c63Jens Axboe continue; 284838bc709279964bdcc64070d4eb2777a0f79bcbbJens Axboe } else 2852866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe break; 28644bbe1093e9ed07e41edd061c4171896a900ad9fJens Axboe } while (ld->queued); 2872866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 28836167d82e5f49dee91c6d2cd426068edee90e36fJens Axboe return ret; 2892866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 2902866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2912866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic int fio_libaio_cancel(struct thread_data *td, struct io_u *io_u) 2922866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 2932866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = td->io_ops->data; 2942866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2952866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe return io_cancel(ld->aio_ctx, &io_u->iocb, ld->aio_events); 2962866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 2972866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2982866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic void fio_libaio_cleanup(struct thread_data *td) 2992866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 3002866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = td->io_ops->data; 3012866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 3022866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe if (ld) { 3032866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe io_destroy(ld->aio_ctx); 3047e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe free(ld->aio_events); 3057e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe free(ld->iocbs); 3067e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe free(ld->io_us); 3072866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe free(ld); 3082866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe } 3092866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 3102866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 3112866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic int fio_libaio_init(struct thread_data *td) 3122866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 313c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe struct libaio_options *o = td->eo; 314241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe struct libaio_data *ld; 315c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe int err = 0; 3162866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 317241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ld = calloc(1, sizeof(*ld)); 318c1db2dced4750f2f506cf4df232dcc75ae05b950Jens Axboe 319c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe /* 320c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe * First try passing in 0 for queue depth, since we don't 321c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe * care about the user ring. If that fails, the kernel is too old 322c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe * and we need the right depth. 323c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe */ 324c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe if (!o->userspace_reap) 325c224ec052b3a5be4fba8da777dd7ccbee2fb34c1Jens Axboe err = io_queue_init(INT_MAX, &ld->aio_ctx); 326c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe if (o->userspace_reap || err == -EINVAL) 327c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe err = io_queue_init(td->o.iodepth, &ld->aio_ctx); 328c1db2dced4750f2f506cf4df232dcc75ae05b950Jens Axboe if (err) { 329c1db2dced4750f2f506cf4df232dcc75ae05b950Jens Axboe td_verror(td, -err, "io_queue_init"); 33075de55acafe8663389449517621da67347354d62Jens Axboe log_err("fio: check /proc/sys/fs/aio-max-nr\n"); 331cb781c758e5df7ff4c6f655c1e4f9df3043a4be9Jens Axboe free(ld); 3322866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe return 1; 3332866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe } 3342866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 335241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ld->entries = td->o.iodepth; 336241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ld->is_pow2 = is_power_of_2(ld->entries); 337241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ld->aio_events = calloc(ld->entries, sizeof(struct io_event)); 338241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ld->iocbs = calloc(ld->entries, sizeof(struct iocb *)); 339241eb089a2fdb1ef0b12706aa15948ceecdba015Jens Axboe ld->io_us = calloc(ld->entries, sizeof(struct io_u *)); 340755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 3412866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe td->io_ops->data = ld; 3422866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe return 0; 3432866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 3442866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 3455f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboestatic struct ioengine_ops ioengine = { 346de890a1e48d40238dac69f302708dde8719de240Steven Lang .name = "libaio", 347de890a1e48d40238dac69f302708dde8719de240Steven Lang .version = FIO_IOOPS_VERSION, 348de890a1e48d40238dac69f302708dde8719de240Steven Lang .init = fio_libaio_init, 349de890a1e48d40238dac69f302708dde8719de240Steven Lang .prep = fio_libaio_prep, 350de890a1e48d40238dac69f302708dde8719de240Steven Lang .queue = fio_libaio_queue, 351de890a1e48d40238dac69f302708dde8719de240Steven Lang .commit = fio_libaio_commit, 352de890a1e48d40238dac69f302708dde8719de240Steven Lang .cancel = fio_libaio_cancel, 353de890a1e48d40238dac69f302708dde8719de240Steven Lang .getevents = fio_libaio_getevents, 354de890a1e48d40238dac69f302708dde8719de240Steven Lang .event = fio_libaio_event, 355de890a1e48d40238dac69f302708dde8719de240Steven Lang .cleanup = fio_libaio_cleanup, 356de890a1e48d40238dac69f302708dde8719de240Steven Lang .open_file = generic_open_file, 357de890a1e48d40238dac69f302708dde8719de240Steven Lang .close_file = generic_close_file, 358de890a1e48d40238dac69f302708dde8719de240Steven Lang .get_file_size = generic_get_file_size, 359de890a1e48d40238dac69f302708dde8719de240Steven Lang .options = options, 360de890a1e48d40238dac69f302708dde8719de240Steven Lang .option_struct_size = sizeof(struct libaio_options), 3612866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe}; 36234cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe 3635f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboestatic void fio_init fio_libaio_register(void) 3645f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe{ 3655f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe register_ioengine(&ioengine); 3665f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe} 3675f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe 3685f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboestatic void fio_exit fio_libaio_unregister(void) 3695f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe{ 3705f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe unregister_ioengine(&ioengine); 3715f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe} 372