libaio.c revision 0cc6ee59fd3caf08e934438241afa4982a15642c
12866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe/* 2da751ca9665bcdeca56d2eec5b629a0953c07662Jens Axboe * libaio engine 3da751ca9665bcdeca56d2eec5b629a0953c07662Jens Axboe * 4da751ca9665bcdeca56d2eec5b629a0953c07662Jens Axboe * IO engine using the Linux native aio interface. 52866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe * 62866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe */ 72866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <stdio.h> 82866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <stdlib.h> 92866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <unistd.h> 102866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <errno.h> 112866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <assert.h> 125f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe 135f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe#include "../fio.h" 142866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 1534cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe#ifdef FIO_HAVE_LIBAIO 1634cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe 172866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestruct libaio_data { 182866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe io_context_t aio_ctx; 192866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct io_event *aio_events; 20755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe struct iocb **iocbs; 217e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe struct io_u **io_us; 22755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe int iocbs_nr; 232866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe}; 242866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 25de890a1e48d40238dac69f302708dde8719de240Steven Langstruct libaio_options { 26de890a1e48d40238dac69f302708dde8719de240Steven Lang struct thread_data *td; 27de890a1e48d40238dac69f302708dde8719de240Steven Lang unsigned int userspace_reap; 28de890a1e48d40238dac69f302708dde8719de240Steven Lang}; 29de890a1e48d40238dac69f302708dde8719de240Steven Lang 30de890a1e48d40238dac69f302708dde8719de240Steven Langstatic struct fio_option options[] = { 31de890a1e48d40238dac69f302708dde8719de240Steven Lang { 32de890a1e48d40238dac69f302708dde8719de240Steven Lang .name = "userspace_reap", 33de890a1e48d40238dac69f302708dde8719de240Steven Lang .type = FIO_OPT_STR_SET, 34de890a1e48d40238dac69f302708dde8719de240Steven Lang .off1 = offsetof(struct libaio_options, userspace_reap), 35de890a1e48d40238dac69f302708dde8719de240Steven Lang .help = "Use alternative user-space reap implementation", 36de890a1e48d40238dac69f302708dde8719de240Steven Lang }, 37de890a1e48d40238dac69f302708dde8719de240Steven Lang { 38de890a1e48d40238dac69f302708dde8719de240Steven Lang .name = NULL, 39de890a1e48d40238dac69f302708dde8719de240Steven Lang }, 40de890a1e48d40238dac69f302708dde8719de240Steven Lang}; 41de890a1e48d40238dac69f302708dde8719de240Steven Lang 427a16dd027f83d7ff8ff0e72bb6bb12fae046fcbfJens Axboestatic int fio_libaio_prep(struct thread_data fio_unused *td, struct io_u *io_u) 432866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 4453cdc6864f7471b28cc9b40a5314ab43e5b1cb5eJens Axboe struct fio_file *f = io_u->file; 4553cdc6864f7471b28cc9b40a5314ab43e5b1cb5eJens Axboe 462866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe if (io_u->ddir == DDIR_READ) 47cec6b55da1c282b5b91ad346c7804171fccf151eJens Axboe io_prep_pread(&io_u->iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset); 4887dc1ab1b4df7b977f60e3d43533a896e2ee665bJens Axboe else if (io_u->ddir == DDIR_WRITE) 49cec6b55da1c282b5b91ad346c7804171fccf151eJens Axboe io_prep_pwrite(&io_u->iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset); 505f9099ea8adf423d0db01274ef0d7e65629c0e1cJens Axboe else if (ddir_sync(io_u->ddir)) 5187dc1ab1b4df7b977f60e3d43533a896e2ee665bJens Axboe io_prep_fsync(&io_u->iocb, f->fd); 522866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 532866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe return 0; 542866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 552866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 562866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic struct io_u *fio_libaio_event(struct thread_data *td, int event) 572866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 582866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = td->io_ops->data; 59f423479dad8916c758792308863dcd833a1a36c8Jens Axboe struct io_event *ev; 60f423479dad8916c758792308863dcd833a1a36c8Jens Axboe struct io_u *io_u; 612866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 62f423479dad8916c758792308863dcd833a1a36c8Jens Axboe ev = ld->aio_events + event; 630cc6ee59fd3caf08e934438241afa4982a15642cJens Axboe io_u = container_of(ev->obj, struct io_u, iocb); 64f423479dad8916c758792308863dcd833a1a36c8Jens Axboe 65f423479dad8916c758792308863dcd833a1a36c8Jens Axboe if (ev->res != io_u->xfer_buflen) { 66f423479dad8916c758792308863dcd833a1a36c8Jens Axboe if (ev->res > io_u->xfer_buflen) 67f423479dad8916c758792308863dcd833a1a36c8Jens Axboe io_u->error = -ev->res; 68f423479dad8916c758792308863dcd833a1a36c8Jens Axboe else 69f423479dad8916c758792308863dcd833a1a36c8Jens Axboe io_u->resid = io_u->xfer_buflen - ev->res; 70f423479dad8916c758792308863dcd833a1a36c8Jens Axboe } else 71f423479dad8916c758792308863dcd833a1a36c8Jens Axboe io_u->error = 0; 72f423479dad8916c758792308863dcd833a1a36c8Jens Axboe 73f423479dad8916c758792308863dcd833a1a36c8Jens Axboe return io_u; 742866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 752866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 76675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenbergstruct aio_ring { 77675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned id; /** kernel internal index number */ 78675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned nr; /** number of io_events */ 79675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned head; 80675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned tail; 81c44b1ff54402c589e8f07436ec56efb4f8b1ac23Jens Axboe 82675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned magic; 83675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned compat_features; 84675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned incompat_features; 85675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned header_length; /** size of aio_ring */ 86675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 87675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg struct io_event events[0]; 88675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg}; 89675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 90675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg#define AIO_RING_MAGIC 0xa10a10a1 91675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 92675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenbergstatic int user_io_getevents(io_context_t aio_ctx, unsigned int max, 93c44b1ff54402c589e8f07436ec56efb4f8b1ac23Jens Axboe struct io_event *events) 94675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg{ 95675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg long i = 0; 96675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg unsigned head; 97c44b1ff54402c589e8f07436ec56efb4f8b1ac23Jens Axboe struct aio_ring *ring = (struct aio_ring*) aio_ctx; 98675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 99675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg while (i < max) { 100675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg head = ring->head; 101675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 102675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg if (head == ring->tail) { 103675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg /* There are no more completions */ 104675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg break; 105675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg } else { 106675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg /* There is another completion to reap */ 107675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg events[i] = ring->events[head]; 108675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg read_barrier(); 109c44b1ff54402c589e8f07436ec56efb4f8b1ac23Jens Axboe ring->head = (head + 1) % ring->nr; 110675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg i++; 111675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg } 112675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg } 113675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 114675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg return i; 115675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg} 116675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg 117e7d2e61694c62b90a2fb84c012b4edcc1973d72cJens Axboestatic int fio_libaio_getevents(struct thread_data *td, unsigned int min, 118e7d2e61694c62b90a2fb84c012b4edcc1973d72cJens Axboe unsigned int max, struct timespec *t) 1192866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 1202866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = td->io_ops->data; 121de890a1e48d40238dac69f302708dde8719de240Steven Lang struct libaio_options *o = td->eo; 1220b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg unsigned actual_min = td->o.iodepth_batch_complete == 0 ? 0 : min; 1230b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg int r, events = 0; 1242866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 1252866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe do { 126de890a1e48d40238dac69f302708dde8719de240Steven Lang if (o->userspace_reap == 1 127675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg && actual_min == 0 128675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg && ((struct aio_ring *)(ld->aio_ctx))->magic 129675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg == AIO_RING_MAGIC) { 130675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg r = user_io_getevents(ld->aio_ctx, max, 131675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg ld->aio_events + events); 132675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg } else { 133675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg r = io_getevents(ld->aio_ctx, actual_min, 134675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg max, ld->aio_events + events, t); 135675012f04dbd821cc5d22ec56930094b9c8a99e5Dan Ehrenberg } 1360b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg if (r >= 0) 1370b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg events += r; 1380b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg else if (r == -EAGAIN) 1392866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe usleep(100); 1400b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg } while (events < min); 1410b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg 1420b7fdba72ade6f642cc0bf86023a42a408544599Dan Ehrenberg return r < 0 ? r : events; 1432866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 1442866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 1452866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic int fio_libaio_queue(struct thread_data *td, struct io_u *io_u) 1462866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 1472866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = td->io_ops->data; 1482866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 1497101d9c24abec4be58a086d85d6d92ec6e6492e9Jens Axboe fio_ro_check(td, io_u); 1507101d9c24abec4be58a086d85d6d92ec6e6492e9Jens Axboe 1512dc1bbeb58edc85f2829eed6729862c438ea2353Jens Axboe if (ld->iocbs_nr == (int) td->o.iodepth) 152755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe return FIO_Q_BUSY; 153755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 154755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe /* 155755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe * fsync is tricky, since it can fail and we need to do it 156755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe * serialized with other io. the reason is that linux doesn't 157755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe * support aio fsync yet. So return busy for the case where we 158755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe * have pending io, to let fio complete those first. 159755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe */ 160f011531e61ae750cdf82074e0dea1379b07fa239Jens Axboe if (ddir_sync(io_u->ddir)) { 161755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe if (ld->iocbs_nr) 162755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe return FIO_Q_BUSY; 1635f9099ea8adf423d0db01274ef0d7e65629c0e1cJens Axboe 164f011531e61ae750cdf82074e0dea1379b07fa239Jens Axboe do_io_u_sync(td, io_u); 1655f9099ea8adf423d0db01274ef0d7e65629c0e1cJens Axboe return FIO_Q_COMPLETED; 166755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe } 167755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 168a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe if (io_u->ddir == DDIR_TRIM) { 169a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe if (ld->iocbs_nr) 170a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe return FIO_Q_BUSY; 171a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe 172a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe do_io_u_trim(td, io_u); 173a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe return FIO_Q_COMPLETED; 174a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe } 175a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe 176755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe ld->iocbs[ld->iocbs_nr] = &io_u->iocb; 1777e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe ld->io_us[ld->iocbs_nr] = io_u; 178755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe ld->iocbs_nr++; 179755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe return FIO_Q_QUEUED; 180755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe} 181755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 1827e77dd026d85253936aef432ba8f3e89b96b805cJens Axboestatic void fio_libaio_queued(struct thread_data *td, struct io_u **io_us, 1837e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe unsigned int nr) 1847e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe{ 1857e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe struct timeval now; 1867e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe unsigned int i; 1877e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe 18812d9d841526ad75a67bb43a90edeefd05f85f11eJens Axboe if (!fio_fill_issue_time(td)) 18912d9d841526ad75a67bb43a90edeefd05f85f11eJens Axboe return; 19012d9d841526ad75a67bb43a90edeefd05f85f11eJens Axboe 1917e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe fio_gettime(&now, NULL); 1927e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe 1937e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe for (i = 0; i < nr; i++) { 1947e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe struct io_u *io_u = io_us[i]; 1957e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe 1967e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe memcpy(&io_u->issue_time, &now, sizeof(now)); 1977e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe io_u_queued(td, io_u); 1987e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe } 1997e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe} 2007e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe 201755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboestatic int fio_libaio_commit(struct thread_data *td) 202755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe{ 203755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe struct libaio_data *ld = td->io_ops->data; 204755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe struct iocb **iocbs; 2057e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe struct io_u **io_us; 2065e00c2c4b7e2d5c1e535dbe10ee4b687f9f4b72bJens Axboe int ret; 207755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 208755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe if (!ld->iocbs_nr) 209755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe return 0; 210755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 2117e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe io_us = ld->io_us; 212755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe iocbs = ld->iocbs; 2132866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe do { 2145e00c2c4b7e2d5c1e535dbe10ee4b687f9f4b72bJens Axboe ret = io_submit(ld->aio_ctx, ld->iocbs_nr, iocbs); 2155e00c2c4b7e2d5c1e535dbe10ee4b687f9f4b72bJens Axboe if (ret > 0) { 2167e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe fio_libaio_queued(td, io_us, ret); 217838bc709279964bdcc64070d4eb2777a0f79bcbbJens Axboe io_u_mark_submit(td, ret); 2185e00c2c4b7e2d5c1e535dbe10ee4b687f9f4b72bJens Axboe ld->iocbs_nr -= ret; 2197e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe io_us += ret; 220755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe iocbs += ret; 2215e00c2c4b7e2d5c1e535dbe10ee4b687f9f4b72bJens Axboe ret = 0; 222838bc709279964bdcc64070d4eb2777a0f79bcbbJens Axboe } else if (!ret || ret == -EAGAIN || ret == -EINTR) { 223838bc709279964bdcc64070d4eb2777a0f79bcbbJens Axboe if (!ret) 224838bc709279964bdcc64070d4eb2777a0f79bcbbJens Axboe io_u_mark_submit(td, ret); 2252866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe continue; 226838bc709279964bdcc64070d4eb2777a0f79bcbbJens Axboe } else 2272866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe break; 2285e00c2c4b7e2d5c1e535dbe10ee4b687f9f4b72bJens Axboe } while (ld->iocbs_nr); 2292866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 23036167d82e5f49dee91c6d2cd426068edee90e36fJens Axboe return ret; 2312866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 2322866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2332866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic int fio_libaio_cancel(struct thread_data *td, struct io_u *io_u) 2342866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 2352866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = td->io_ops->data; 2362866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2372866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe return io_cancel(ld->aio_ctx, &io_u->iocb, ld->aio_events); 2382866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 2392866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2402866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic void fio_libaio_cleanup(struct thread_data *td) 2412866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 2422866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = td->io_ops->data; 2432866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2442866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe if (ld) { 2452866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe io_destroy(ld->aio_ctx); 2467e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe free(ld->aio_events); 2477e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe free(ld->iocbs); 2487e77dd026d85253936aef432ba8f3e89b96b805cJens Axboe free(ld->io_us); 2492866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe free(ld); 2502866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe } 2512866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 2522866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2532866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic int fio_libaio_init(struct thread_data *td) 2542866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 2552866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct libaio_data *ld = malloc(sizeof(*ld)); 256c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe struct libaio_options *o = td->eo; 257c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe int err = 0; 2582866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2592866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe memset(ld, 0, sizeof(*ld)); 260c1db2dced4750f2f506cf4df232dcc75ae05b950Jens Axboe 261c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe /* 262c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe * First try passing in 0 for queue depth, since we don't 263c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe * care about the user ring. If that fails, the kernel is too old 264c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe * and we need the right depth. 265c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe */ 266c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe if (!o->userspace_reap) 267c224ec052b3a5be4fba8da777dd7ccbee2fb34c1Jens Axboe err = io_queue_init(INT_MAX, &ld->aio_ctx); 268c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe if (o->userspace_reap || err == -EINVAL) 269c90e1017b9384b170d47b97d2353f7bff3b5a34dJens Axboe err = io_queue_init(td->o.iodepth, &ld->aio_ctx); 270c1db2dced4750f2f506cf4df232dcc75ae05b950Jens Axboe if (err) { 271c1db2dced4750f2f506cf4df232dcc75ae05b950Jens Axboe td_verror(td, -err, "io_queue_init"); 27275de55acafe8663389449517621da67347354d62Jens Axboe log_err("fio: check /proc/sys/fs/aio-max-nr\n"); 273cb781c758e5df7ff4c6f655c1e4f9df3043a4be9Jens Axboe free(ld); 2742866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe return 1; 2752866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe } 2762866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2772dc1bbeb58edc85f2829eed6729862c438ea2353Jens Axboe ld->aio_events = malloc(td->o.iodepth * sizeof(struct io_event)); 2782dc1bbeb58edc85f2829eed6729862c438ea2353Jens Axboe memset(ld->aio_events, 0, td->o.iodepth * sizeof(struct io_event)); 2792dc1bbeb58edc85f2829eed6729862c438ea2353Jens Axboe ld->iocbs = malloc(td->o.iodepth * sizeof(struct iocb *)); 280755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe memset(ld->iocbs, 0, sizeof(struct iocb *)); 2812dc1bbeb58edc85f2829eed6729862c438ea2353Jens Axboe ld->io_us = malloc(td->o.iodepth * sizeof(struct io_u *)); 2822dc1bbeb58edc85f2829eed6729862c438ea2353Jens Axboe memset(ld->io_us, 0, td->o.iodepth * sizeof(struct io_u *)); 283755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe ld->iocbs_nr = 0; 284755200a326a33e5e19b16dfd5e013dd98bcf1916Jens Axboe 2852866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe td->io_ops->data = ld; 2862866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe return 0; 2872866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 2882866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2895f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboestatic struct ioengine_ops ioengine = { 290de890a1e48d40238dac69f302708dde8719de240Steven Lang .name = "libaio", 291de890a1e48d40238dac69f302708dde8719de240Steven Lang .version = FIO_IOOPS_VERSION, 292de890a1e48d40238dac69f302708dde8719de240Steven Lang .init = fio_libaio_init, 293de890a1e48d40238dac69f302708dde8719de240Steven Lang .prep = fio_libaio_prep, 294de890a1e48d40238dac69f302708dde8719de240Steven Lang .queue = fio_libaio_queue, 295de890a1e48d40238dac69f302708dde8719de240Steven Lang .commit = fio_libaio_commit, 296de890a1e48d40238dac69f302708dde8719de240Steven Lang .cancel = fio_libaio_cancel, 297de890a1e48d40238dac69f302708dde8719de240Steven Lang .getevents = fio_libaio_getevents, 298de890a1e48d40238dac69f302708dde8719de240Steven Lang .event = fio_libaio_event, 299de890a1e48d40238dac69f302708dde8719de240Steven Lang .cleanup = fio_libaio_cleanup, 300de890a1e48d40238dac69f302708dde8719de240Steven Lang .open_file = generic_open_file, 301de890a1e48d40238dac69f302708dde8719de240Steven Lang .close_file = generic_close_file, 302de890a1e48d40238dac69f302708dde8719de240Steven Lang .get_file_size = generic_get_file_size, 303de890a1e48d40238dac69f302708dde8719de240Steven Lang .options = options, 304de890a1e48d40238dac69f302708dde8719de240Steven Lang .option_struct_size = sizeof(struct libaio_options), 3052866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe}; 30634cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe 30734cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe#else /* FIO_HAVE_LIBAIO */ 30834cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe 30934cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe/* 31034cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe * When we have a proper configure system in place, we simply wont build 31134cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe * and install this io engine. For now install a crippled version that 31234cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe * just complains and fails to load. 31334cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe */ 31434cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboestatic int fio_libaio_init(struct thread_data fio_unused *td) 31534cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe{ 316a3edaf76b30880c3bf4455c30614ce88cdbbe071Jens Axboe log_err("fio: libaio not available\n"); 31734cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe return 1; 31834cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe} 31934cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe 3205f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboestatic struct ioengine_ops ioengine = { 32134cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe .name = "libaio", 32234cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe .version = FIO_IOOPS_VERSION, 32334cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe .init = fio_libaio_init, 32434cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe}; 32534cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe 32634cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe#endif 3275f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe 3285f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboestatic void fio_init fio_libaio_register(void) 3295f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe{ 3305f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe register_ioengine(&ioengine); 3315f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe} 3325f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe 3335f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboestatic void fio_exit fio_libaio_unregister(void) 3345f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe{ 3355f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe unregister_ioengine(&ioengine); 3365f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe} 337