libaio.c revision c90e1017b9384b170d47b97d2353f7bff3b5a34d
12866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe/* 2da751ca9665bcdeca56d2eec5b629a0953c07662Jens Axboe * libaio engine 3da751ca9665bcdeca56d2eec5b629a0953c07662Jens Axboe * 4da751ca9665bcdeca56d2eec5b629a0953c07662Jens Axboe * IO engine using the Linux native aio interface. 52866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe * 62866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe */ 72866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <stdio.h> 82866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <stdlib.h> 92866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <unistd.h> 102866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <errno.h> 112866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <assert.h> 125f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe 135f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe#include "../fio.h" 142866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 1534cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe#ifdef FIO_HAVE_LIBAIO 1634cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe 172866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#define ev_to_iou(ev) (struct io_u *) ((unsigned long) (ev)->obj) 182866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 192866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestruct libaio_data { 202866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe io_context_t aio_ctx; 212866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct io_event *aio_events; 22755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe struct iocb **iocbs; 237e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe struct io_u **io_us; 24755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe int iocbs_nr; 252866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe}; 262866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 27de890a1e48d40238dac69f302708dde8719de240Steven Langstruct libaio_options { 28de890a1e48d40238dac69f302708dde8719de240Steven Lang struct thread_data *td; 29de890a1e48d40238dac69f302708dde8719de240Steven Lang unsigned int userspace_reap; 30de890a1e48d40238dac69f302708dde8719de240Steven Lang}; 31de890a1e48d40238dac69f302708dde8719de240Steven Lang 32de890a1e48d40238dac69f302708dde8719de240Steven Langstatic struct fio_option options[] = { 33de890a1e48d40238dac69f302708dde8719de240Steven Lang { 34de890a1e48d40238dac69f302708dde8719de240Steven Lang .name = "userspace_reap", 35de890a1e48d40238dac69f302708dde8719de240Steven Lang .type = FIO_OPT_STR_SET, 36de890a1e48d40238dac69f302708dde8719de240Steven Lang .off1 = offsetof(struct libaio_options, userspace_reap), 37de890a1e48d40238dac69f302708dde8719de240Steven Lang .help = "Use alternative user-space reap implementation", 38de890a1e48d40238dac69f302708dde8719de240Steven Lang }, 39de890a1e48d40238dac69f302708dde8719de240Steven Lang { 40de890a1e48d40238dac69f302708dde8719de240Steven Lang .name = NULL, 41de890a1e48d40238dac69f302708dde8719de240Steven Lang }, 42de890a1e48d40238dac69f302708dde8719de240Steven Lang}; 43de890a1e48d40238dac69f302708dde8719de240Steven Lang 447a16dd027f83d7ff8ff0e72bb6bb12fae046fcbfJens Axboestatic int fio_libaio_prep(struct thread_data fio_unused *td, struct io_u *io_u) 452866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 4653cdc6864f7471b28cc9b40a5314ab43e5b1cb5eJens Axboe struct fio_file *f = io_u->file; 4753cdc6864f7471b28cc9b40a5314ab43e5b1cb5eJens Axboe 482866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe if (io_u->ddir == DDIR_READ) 49cec6b55da1c282b5b91ad346c7804171fccf151eJens Axboe io_prep_pread(&io_u->iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset); 5087dc1ab1b4df7b977f60e3d43533a896e2ee665bJens Axboe else if (io_u->ddir == DDIR_WRITE) 51cec6b55da1c282b5b91ad346c7804171fccf151eJens Axboe io_prep_pwrite(&io_u->iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset); 525f9099ea8adf423d0db01274ef0d7e65629c0e1cJens Axboe else if (ddir_sync(io_u->ddir)) 5387dc1ab1b4df7b977f60e3d43533a896e2ee665bJens Axboe io_prep_fsync(&io_u->iocb, f->fd); 542866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 552866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe return 0; 562866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 572866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 582866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic struct io_u *fio_libaio_event(struct thread_data *td, int event) 592866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 602866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = td->io_ops->data; 61f423479dad8916c758792308863dcd833a1a36c8Jens Axboe struct io_event *ev; 62f423479dad8916c758792308863dcd833a1a36c8Jens Axboe struct io_u *io_u; 632866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 64f423479dad8916c758792308863dcd833a1a36c8Jens Axboe ev = ld->aio_events + event; 65f423479dad8916c758792308863dcd833a1a36c8Jens Axboe io_u = ev_to_iou(ev); 66f423479dad8916c758792308863dcd833a1a36c8Jens Axboe 67f423479dad8916c758792308863dcd833a1a36c8Jens Axboe if (ev->res != io_u->xfer_buflen) { 68f423479dad8916c758792308863dcd833a1a36c8Jens Axboe if (ev->res > io_u->xfer_buflen) 69f423479dad8916c758792308863dcd833a1a36c8Jens Axboe io_u->error = -ev->res; 70f423479dad8916c758792308863dcd833a1a36c8Jens Axboe else 71f423479dad8916c758792308863dcd833a1a36c8Jens Axboe io_u->resid = io_u->xfer_buflen - ev->res; 72f423479dad8916c758792308863dcd833a1a36c8Jens Axboe } else 73f423479dad8916c758792308863dcd833a1a36c8Jens Axboe io_u->error = 0; 74f423479dad8916c758792308863dcd833a1a36c8Jens Axboe 75f423479dad8916c758792308863dcd833a1a36c8Jens Axboe return io_u; 762866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 772866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 78675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenbergstruct aio_ring { 79675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned id; /** kernel internal index number */ 80675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned nr; /** number of io_events */ 81675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned head; 82675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned tail; 83c44b1ff54402c589e8f07436ec56efb4f8b1ac23Jens Axboe 84675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned magic; 85675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned compat_features; 86675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned incompat_features; 87675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned header_length; /** size of aio_ring */ 88675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 89675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg struct io_event events[0]; 90675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg}; 91675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 92675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg#define AIO_RING_MAGIC 0xa10a10a1 93675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 94675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenbergstatic int user_io_getevents(io_context_t aio_ctx, unsigned int max, 95c44b1ff54402c589e8f07436ec56efb4f8b1ac23Jens Axboe struct io_event *events) 96675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg{ 97675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg long i = 0; 98675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned head; 99c44b1ff54402c589e8f07436ec56efb4f8b1ac23Jens Axboe struct aio_ring *ring = (struct aio_ring*) aio_ctx; 100675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 101675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg while (i < max) { 102675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg head = ring->head; 103675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 104675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg if (head == ring->tail) { 105675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg /* There are no more completions */ 106675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg break; 107675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg } else { 108675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg /* There is another completion to reap */ 109675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg events[i] = ring->events[head]; 110675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg read_barrier(); 111c44b1ff54402c589e8f07436ec56efb4f8b1ac23Jens Axboe ring->head = (head + 1) % ring->nr; 112675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg i++; 113675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg } 114675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg } 115675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 116675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg return i; 117675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg} 118675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 119e7d2e61694c62b90a2fb84c012b4edcc1973d72cJens Axboestatic int fio_libaio_getevents(struct thread_data *td, unsigned int min, 120e7d2e61694c62b90a2fb84c012b4edcc1973d72cJens Axboe unsigned int max, struct timespec *t) 1212866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 1222866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = td->io_ops->data; 123de890a1e48d40238dac69f302708dde8719de240Steven Lang struct libaio_options *o = td->eo; 1240b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg unsigned actual_min = td->o.iodepth_batch_complete == 0 ? 0 : min; 1250b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg int r, events = 0; 1262866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 1272866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe do { 128de890a1e48d40238dac69f302708dde8719de240Steven Lang if (o->userspace_reap == 1 129675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg && actual_min == 0 130675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg && ((struct aio_ring *)(ld->aio_ctx))->magic 131675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg == AIO_RING_MAGIC) { 132675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg r = user_io_getevents(ld->aio_ctx, max, 133675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg ld->aio_events + events); 134675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg } else { 135675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg r = io_getevents(ld->aio_ctx, actual_min, 136675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg max, ld->aio_events + events, t); 137675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg } 1380b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg if (r >= 0) 1390b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg events += r; 1400b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg else if (r == -EAGAIN) 1412866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe usleep(100); 1420b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg } while (events < min); 1430b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg 1440b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg return r < 0 ? r : events; 1452866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 1462866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 1472866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic int fio_libaio_queue(struct thread_data *td, struct io_u *io_u) 1482866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 1492866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = td->io_ops->data; 1502866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 1517101d9c24abec4be58a086d85d6d92ec6e6492e9Jens Axboe fio_ro_check(td, io_u); 1527101d9c24abec4be58a086d85d6d92ec6e6492e9Jens Axboe 1532dc1bbeb58edc85f2829eed6729862c438ea2353Jens Axboe if (ld->iocbs_nr == (int) td->o.iodepth) 154755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe return FIO_Q_BUSY; 155755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 156755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe /* 157755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe * fsync is tricky, since it can fail and we need to do it 158755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe * serialized with other io. the reason is that linux doesn't 159755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe * support aio fsync yet. So return busy for the case where we 160755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe * have pending io, to let fio complete those first. 161755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe */ 162f011531e61ae750cdf82074e0dea1379b07fa239Jens Axboe if (ddir_sync(io_u->ddir)) { 163755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe if (ld->iocbs_nr) 164755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe return FIO_Q_BUSY; 1655f9099ea8adf423d0db01274ef0d7e65629c0e1cJens Axboe 166f011531e61ae750cdf82074e0dea1379b07fa239Jens Axboe do_io_u_sync(td, io_u); 1675f9099ea8adf423d0db01274ef0d7e65629c0e1cJens Axboe return FIO_Q_COMPLETED; 168755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe } 169755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 170a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe if (io_u->ddir == DDIR_TRIM) { 171a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe if (ld->iocbs_nr) 172a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe return FIO_Q_BUSY; 173a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe 174a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe do_io_u_trim(td, io_u); 175a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe return FIO_Q_COMPLETED; 176a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe } 177a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe 178755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe ld->iocbs[ld->iocbs_nr] = &io_u->iocb; 1797e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe ld->io_us[ld->iocbs_nr] = io_u; 180755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe ld->iocbs_nr++; 181755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe return FIO_Q_QUEUED; 182755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe} 183755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 1847e77dd026d85253936aef432ba8f3e89b96b805cJens Axboestatic void fio_libaio_queued(struct thread_data *td, struct io_u **io_us, 1857e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe unsigned int nr) 1867e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe{ 1877e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe struct timeval now; 1887e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe unsigned int i; 1897e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe 19012d9d841526ad75a67bb43a90edeefd05f85f11eJens Axboe if (!fio_fill_issue_time(td)) 19112d9d841526ad75a67bb43a90edeefd05f85f11eJens Axboe return; 19212d9d841526ad75a67bb43a90edeefd05f85f11eJens Axboe 1937e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe fio_gettime(&now, NULL); 1947e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe 1957e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe for (i = 0; i < nr; i++) { 1967e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe struct io_u *io_u = io_us[i]; 1977e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe 1987e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe memcpy(&io_u->issue_time, &now, sizeof(now)); 1997e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe io_u_queued(td, io_u); 2007e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe } 2017e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe} 2027e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe 203755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboestatic int fio_libaio_commit(struct thread_data *td) 204755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe{ 205755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe struct libaio_data *ld = td->io_ops->data; 206755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe struct iocb **iocbs; 2077e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe struct io_u **io_us; 2085e00c2c4b7e2d5c1e535dbe10ee4b687f9f4b72bJens Axboe int ret; 209755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 210755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe if (!ld->iocbs_nr) 211755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe return 0; 212755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 2137e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe io_us = ld->io_us; 214755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe iocbs = ld->iocbs; 2152866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe do { 2165e00c2c4b7e2d5c1e535dbe10ee4b687f9f4b72bJens Axboe ret = io_submit(ld->aio_ctx, ld->iocbs_nr, iocbs); 2175e00c2c4b7e2d5c1e535dbe10ee4b687f9f4b72bJens Axboe if (ret > 0) { 2187e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe fio_libaio_queued(td, io_us, ret); 219838bc709279964bdcc64070d4eb2777a0f79bcbbJens Axboe io_u_mark_submit(td, ret); 2205e00c2c4b7e2d5c1e535dbe10ee4b687f9f4b72bJens Axboe ld->iocbs_nr -= ret; 2217e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe io_us += ret; 222755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe iocbs += ret; 2235e00c2c4b7e2d5c1e535dbe10ee4b687f9f4b72bJens Axboe ret = 0; 224838bc709279964bdcc64070d4eb2777a0f79bcbbJens Axboe } else if (!ret || ret == -EAGAIN || ret == -EINTR) { 225838bc709279964bdcc64070d4eb2777a0f79bcbbJens Axboe if (!ret) 226838bc709279964bdcc64070d4eb2777a0f79bcbbJens Axboe io_u_mark_submit(td, ret); 2272866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe continue; 228838bc709279964bdcc64070d4eb2777a0f79bcbbJens Axboe } else 2292866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe break; 2305e00c2c4b7e2d5c1e535dbe10ee4b687f9f4b72bJens Axboe } while (ld->iocbs_nr); 2312866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 23236167d82e5f49dee91c6d2cd426068edee90e36fJens Axboe return ret; 2332866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 2342866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2352866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic int fio_libaio_cancel(struct thread_data *td, struct io_u *io_u) 2362866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 2372866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = td->io_ops->data; 2382866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2392866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe return io_cancel(ld->aio_ctx, &io_u->iocb, ld->aio_events); 2402866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 2412866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2422866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic void fio_libaio_cleanup(struct thread_data *td) 2432866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 2442866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = td->io_ops->data; 2452866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2462866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe if (ld) { 2472866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe io_destroy(ld->aio_ctx); 2487e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe free(ld->aio_events); 2497e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe free(ld->iocbs); 2507e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe free(ld->io_us); 2512866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe free(ld); 2522866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe } 2532866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 2542866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2552866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic int fio_libaio_init(struct thread_data *td) 2562866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 2572866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = malloc(sizeof(*ld)); 258c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe struct libaio_options *o = td->eo; 259c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe int err = 0; 2602866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2612866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe memset(ld, 0, sizeof(*ld)); 262c1db2dced4750f2f506cf4df232dcc75ae05b950Jens Axboe 263c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe /* 264c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe * First try passing in 0 for queue depth, since we don't 265c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe * care about the user ring. If that fails, the kernel is too old 266c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe * and we need the right depth. 267c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe */ 268c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe if (!o->userspace_reap) 269c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe err = io_queue_init(0, &ld->aio_ctx); 270c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe if (o->userspace_reap || err == -EINVAL) 271c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe err = io_queue_init(td->o.iodepth, &ld->aio_ctx); 272c1db2dced4750f2f506cf4df232dcc75ae05b950Jens Axboe if (err) { 273c1db2dced4750f2f506cf4df232dcc75ae05b950Jens Axboe td_verror(td, -err, "io_queue_init"); 27475de55acafe8663389449517621da67347354d62Jens Axboe log_err("fio: check /proc/sys/fs/aio-max-nr\n"); 275cb781c758e5df7ff4c6f655c1e4f9df3043a4be9Jens Axboe free(ld); 2762866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe return 1; 2772866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe } 2782866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2792dc1bbeb58edc85f2829eed6729862c438ea2353Jens Axboe ld->aio_events = malloc(td->o.iodepth * sizeof(struct io_event)); 2802dc1bbeb58edc85f2829eed6729862c438ea2353Jens Axboe memset(ld->aio_events, 0, td->o.iodepth * sizeof(struct io_event)); 2812dc1bbeb58edc85f2829eed6729862c438ea2353Jens Axboe ld->iocbs = malloc(td->o.iodepth * sizeof(struct iocb *)); 282755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe memset(ld->iocbs, 0, sizeof(struct iocb *)); 2832dc1bbeb58edc85f2829eed6729862c438ea2353Jens Axboe ld->io_us = malloc(td->o.iodepth * sizeof(struct io_u *)); 2842dc1bbeb58edc85f2829eed6729862c438ea2353Jens Axboe memset(ld->io_us, 0, td->o.iodepth * sizeof(struct io_u *)); 285755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe ld->iocbs_nr = 0; 286755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 2872866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe td->io_ops->data = ld; 2882866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe return 0; 2892866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 2902866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2915f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboestatic struct ioengine_ops ioengine = { 292de890a1e48d40238dac69f302708dde8719de240Steven Lang .name = "libaio", 293de890a1e48d40238dac69f302708dde8719de240Steven Lang .version = FIO_IOOPS_VERSION, 294de890a1e48d40238dac69f302708dde8719de240Steven Lang .init = fio_libaio_init, 295de890a1e48d40238dac69f302708dde8719de240Steven Lang .prep = fio_libaio_prep, 296de890a1e48d40238dac69f302708dde8719de240Steven Lang .queue = fio_libaio_queue, 297de890a1e48d40238dac69f302708dde8719de240Steven Lang .commit = fio_libaio_commit, 298de890a1e48d40238dac69f302708dde8719de240Steven Lang .cancel = fio_libaio_cancel, 299de890a1e48d40238dac69f302708dde8719de240Steven Lang .getevents = fio_libaio_getevents, 300de890a1e48d40238dac69f302708dde8719de240Steven Lang .event = fio_libaio_event, 301de890a1e48d40238dac69f302708dde8719de240Steven Lang .cleanup = fio_libaio_cleanup, 302de890a1e48d40238dac69f302708dde8719de240Steven Lang .open_file = generic_open_file, 303de890a1e48d40238dac69f302708dde8719de240Steven Lang .close_file = generic_close_file, 304de890a1e48d40238dac69f302708dde8719de240Steven Lang .get_file_size = generic_get_file_size, 305de890a1e48d40238dac69f302708dde8719de240Steven Lang .options = options, 306de890a1e48d40238dac69f302708dde8719de240Steven Lang .option_struct_size = sizeof(struct libaio_options), 3072866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe}; 30834cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe 30934cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe#else /* FIO_HAVE_LIBAIO */ 31034cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe 31134cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe/* 31234cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe * When we have a proper configure system in place, we simply wont build 31334cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe * and install this io engine. For now install a crippled version that 31434cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe * just complains and fails to load. 31534cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe */ 31634cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboestatic int fio_libaio_init(struct thread_data fio_unused *td) 31734cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe{ 318a3edaf76b30880c3bf4455c30614ce88cdbbe071Jens Axboe log_err("fio: libaio not available\n"); 31934cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe return 1; 32034cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe} 32134cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe 3225f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboestatic struct ioengine_ops ioengine = { 32334cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe .name = "libaio", 32434cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe .version = FIO_IOOPS_VERSION, 32534cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe .init = fio_libaio_init, 32634cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe}; 32734cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe 32834cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe#endif 3295f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe 3305f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboestatic void fio_init fio_libaio_register(void) 3315f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe{ 3325f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe register_ioengine(&ioengine); 3335f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe} 3345f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe 3355f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboestatic void fio_exit fio_libaio_unregister(void) 3365f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe{ 3375f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe unregister_ioengine(&ioengine); 3385f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe} 339