12866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe/* 2da751ca9665bcdeca56d2eec5b629a0953c07662Jens Axboe * splice engine 3da751ca9665bcdeca56d2eec5b629a0953c07662Jens Axboe * 4da751ca9665bcdeca56d2eec5b629a0953c07662Jens Axboe * IO engine that transfers data by doing splices to/from pipes and 5da751ca9665bcdeca56d2eec5b629a0953c07662Jens Axboe * the files. 62866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe * 72866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe */ 82866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <stdio.h> 92866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <stdlib.h> 102866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <unistd.h> 112866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <errno.h> 122866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <assert.h> 132866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe#include <sys/poll.h> 1481887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe#include <sys/mman.h> 155f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe 165f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe#include "../fio.h" 172866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 182866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestruct spliceio_data { 192866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe int pipe[2]; 20f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe int vmsplice_to_user; 218b850243258107412710cd38252f0cfb8802593dJens Axboe int vmsplice_to_user_map; 222866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe}; 232866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 242866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe/* 25f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe * vmsplice didn't use to support splicing to user space, this is the old 26f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe * variant of getting that job done. Doesn't make a lot of sense, but it 27f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe * uses splices to move data from the source into a pipe. 282866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe */ 29f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboestatic int fio_splice_read_old(struct thread_data *td, struct io_u *io_u) 302866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 31eda3a60699e1d96bb68875ef2169ca819eb8f4f9Elliott Hughes struct spliceio_data *sd = td->io_ops_data; 3253cdc6864f7471b28cc9b40a5314ab43e5b1cb5eJens Axboe struct fio_file *f = io_u->file; 332866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe int ret, ret2, buflen; 342866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe off_t offset; 352866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe void *p; 362866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 372866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe offset = io_u->offset; 38cec6b55da1c282b5b91ad346c7804171fccf151eJens Axboe buflen = io_u->xfer_buflen; 39cec6b55da1c282b5b91ad346c7804171fccf151eJens Axboe p = io_u->xfer_buf; 402866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe while (buflen) { 412866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe int this_len = buflen; 422866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 432866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe if (this_len > SPLICE_DEF_SIZE) 442866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe this_len = SPLICE_DEF_SIZE; 452866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 4653cdc6864f7471b28cc9b40a5314ab43e5b1cb5eJens Axboe ret = splice(f->fd, &offset, sd->pipe[1], NULL, this_len, SPLICE_F_MORE); 472866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe if (ret < 0) { 482866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe if (errno == ENODATA || errno == EAGAIN) 492866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe continue; 502866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 51b4ba5f302c467dc10c8b89459a8cb97d2958f359Jens Axboe return -errno; 522866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe } 532866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 542866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe buflen -= ret; 552866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 562866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe while (ret) { 572866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe ret2 = read(sd->pipe[0], p, ret); 582866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe if (ret2 < 0) 59b4ba5f302c467dc10c8b89459a8cb97d2958f359Jens Axboe return -errno; 602866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 612866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe ret -= ret2; 622866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe p += ret2; 632866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe } 642866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe } 652866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 66cec6b55da1c282b5b91ad346c7804171fccf151eJens Axboe return io_u->xfer_buflen; 672866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 682866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 692866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe/* 70f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe * We can now vmsplice into userspace, so do the transfer by splicing into 71f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe * a pipe and vmsplicing that into userspace. 72f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe */ 73f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboestatic int fio_splice_read(struct thread_data *td, struct io_u *io_u) 74f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe{ 75eda3a60699e1d96bb68875ef2169ca819eb8f4f9Elliott Hughes struct spliceio_data *sd = td->io_ops_data; 76f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe struct fio_file *f = io_u->file; 77f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe struct iovec iov; 788b850243258107412710cd38252f0cfb8802593dJens Axboe int ret , buflen, mmap_len; 79f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe off_t offset; 8081887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe void *p, *map; 81f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe 828b850243258107412710cd38252f0cfb8802593dJens Axboe ret = 0; 83f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe offset = io_u->offset; 8481887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe mmap_len = buflen = io_u->xfer_buflen; 8581887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe 868b850243258107412710cd38252f0cfb8802593dJens Axboe if (sd->vmsplice_to_user_map) { 878b850243258107412710cd38252f0cfb8802593dJens Axboe map = mmap(io_u->xfer_buf, buflen, PROT_READ, MAP_PRIVATE|OS_MAP_ANON, 0, 0); 888b850243258107412710cd38252f0cfb8802593dJens Axboe if (map == MAP_FAILED) { 898b850243258107412710cd38252f0cfb8802593dJens Axboe td_verror(td, errno, "mmap io_u"); 908b850243258107412710cd38252f0cfb8802593dJens Axboe return -1; 918b850243258107412710cd38252f0cfb8802593dJens Axboe } 928b850243258107412710cd38252f0cfb8802593dJens Axboe 938b850243258107412710cd38252f0cfb8802593dJens Axboe p = map; 948b850243258107412710cd38252f0cfb8802593dJens Axboe } else { 958b850243258107412710cd38252f0cfb8802593dJens Axboe map = NULL; 968b850243258107412710cd38252f0cfb8802593dJens Axboe p = io_u->xfer_buf; 9781887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe } 9881887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe 99f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe while (buflen) { 100f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe int this_len = buflen; 10181887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe int flags = 0; 102f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe 10381887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe if (this_len > SPLICE_DEF_SIZE) { 104f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe this_len = SPLICE_DEF_SIZE; 10581887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe flags = SPLICE_F_MORE; 10681887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe } 107f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe 10881887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe ret = splice(f->fd, &offset, sd->pipe[1], NULL, this_len,flags); 109f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe if (ret < 0) { 110f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe if (errno == ENODATA || errno == EAGAIN) 111f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe continue; 112f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe 11381887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe td_verror(td, errno, "splice-from-fd"); 11481887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe break; 115f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe } 116f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe 117f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe buflen -= ret; 118f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe iov.iov_base = p; 119f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe iov.iov_len = ret; 120f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe 121f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe while (iov.iov_len) { 122f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe ret = vmsplice(sd->pipe[0], &iov, 1, SPLICE_F_MOVE); 12381887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe if (ret < 0) { 124cd98801fbed25dcc49f58cf1451af22f0455a6e6Jens Axboe if (errno == EFAULT && 125cd98801fbed25dcc49f58cf1451af22f0455a6e6Jens Axboe sd->vmsplice_to_user_map) { 1268b850243258107412710cd38252f0cfb8802593dJens Axboe sd->vmsplice_to_user_map = 0; 1278b850243258107412710cd38252f0cfb8802593dJens Axboe munmap(map, mmap_len); 128cd98801fbed25dcc49f58cf1451af22f0455a6e6Jens Axboe map = NULL; 129cd98801fbed25dcc49f58cf1451af22f0455a6e6Jens Axboe p = io_u->xfer_buf; 130cd98801fbed25dcc49f58cf1451af22f0455a6e6Jens Axboe iov.iov_base = p; 131cd98801fbed25dcc49f58cf1451af22f0455a6e6Jens Axboe continue; 1328b850243258107412710cd38252f0cfb8802593dJens Axboe } 1333a6d267d644226983a430cf0f217c27f4c38cd45Jens Axboe if (errno == EBADF) { 1343a6d267d644226983a430cf0f217c27f4c38cd45Jens Axboe ret = -EBADF; 1353a6d267d644226983a430cf0f217c27f4c38cd45Jens Axboe break; 1363a6d267d644226983a430cf0f217c27f4c38cd45Jens Axboe } 13781887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe td_verror(td, errno, "vmsplice"); 13881887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe break; 13981887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe } else if (!ret) { 14081887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe td_verror(td, ENODATA, "vmsplice"); 14181887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe ret = -1; 14281887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe break; 14381887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe } 144f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe 145f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe iov.iov_len -= ret; 146f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe iov.iov_base += ret; 147cd98801fbed25dcc49f58cf1451af22f0455a6e6Jens Axboe p += ret; 148f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe } 14981887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe if (ret < 0) 15081887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe break; 15181887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe } 15281887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe 1538b850243258107412710cd38252f0cfb8802593dJens Axboe if (sd->vmsplice_to_user_map && munmap(map, mmap_len) < 0) { 15481887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe td_verror(td, errno, "munnap io_u"); 15581887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe return -1; 156f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe } 15781887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe if (ret < 0) 15881887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe return ret; 159f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe 160f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe return io_u->xfer_buflen; 161f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe} 162f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe 163f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe/* 1642866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe * For splice writing, we can vmsplice our data buffer directly into a 1652866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe * pipe and then splice that to a file. 1662866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe */ 1672866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic int fio_splice_write(struct thread_data *td, struct io_u *io_u) 1682866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 169eda3a60699e1d96bb68875ef2169ca819eb8f4f9Elliott Hughes struct spliceio_data *sd = td->io_ops_data; 170f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe struct iovec iov = { 171f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe .iov_base = io_u->xfer_buf, 172f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe .iov_len = io_u->xfer_buflen, 1732866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe }; 1742866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct pollfd pfd = { .fd = sd->pipe[1], .events = POLLOUT, }; 17553cdc6864f7471b28cc9b40a5314ab43e5b1cb5eJens Axboe struct fio_file *f = io_u->file; 1762866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe off_t off = io_u->offset; 1772866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe int ret, ret2; 1782866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 179f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe while (iov.iov_len) { 1802866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe if (poll(&pfd, 1, -1) < 0) 1812866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe return errno; 1822866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 183f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe ret = vmsplice(sd->pipe[1], &iov, 1, SPLICE_F_NONBLOCK); 1842866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe if (ret < 0) 185b4ba5f302c467dc10c8b89459a8cb97d2958f359Jens Axboe return -errno; 1862866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 187f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe iov.iov_len -= ret; 188f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe iov.iov_base += ret; 1892866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 1902866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe while (ret) { 19153cdc6864f7471b28cc9b40a5314ab43e5b1cb5eJens Axboe ret2 = splice(sd->pipe[0], NULL, f->fd, &off, ret, 0); 1922866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe if (ret2 < 0) 193b4ba5f302c467dc10c8b89459a8cb97d2958f359Jens Axboe return -errno; 1942866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 1952866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe ret -= ret2; 1962866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe } 1972866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe } 1982866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 199cec6b55da1c282b5b91ad346c7804171fccf151eJens Axboe return io_u->xfer_buflen; 2002866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 2012866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2022866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic int fio_spliceio_queue(struct thread_data *td, struct io_u *io_u) 2032866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 204eda3a60699e1d96bb68875ef2169ca819eb8f4f9Elliott Hughes struct spliceio_data *sd = td->io_ops_data; 20524d23ca76f9be4500270e7074a1dab038e3a4a2bJens Axboe int ret = 0; 2062866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2077101d9c24abec4be58a086d85d6d92ec6e6492e9Jens Axboe fio_ro_check(td, io_u); 2087101d9c24abec4be58a086d85d6d92ec6e6492e9Jens Axboe 209f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe if (io_u->ddir == DDIR_READ) { 210436885758c1b859392b80c616c338f9cfe9a9dfbJens Axboe if (sd->vmsplice_to_user) { 211f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe ret = fio_splice_read(td, io_u); 212436885758c1b859392b80c616c338f9cfe9a9dfbJens Axboe /* 213436885758c1b859392b80c616c338f9cfe9a9dfbJens Axboe * This kernel doesn't support vmsplice to user 214436885758c1b859392b80c616c338f9cfe9a9dfbJens Axboe * space. Reset the vmsplice_to_user flag, so that 215436885758c1b859392b80c616c338f9cfe9a9dfbJens Axboe * we retry below and don't hit this path again. 216436885758c1b859392b80c616c338f9cfe9a9dfbJens Axboe */ 217436885758c1b859392b80c616c338f9cfe9a9dfbJens Axboe if (ret == -EBADF) 218436885758c1b859392b80c616c338f9cfe9a9dfbJens Axboe sd->vmsplice_to_user = 0; 219436885758c1b859392b80c616c338f9cfe9a9dfbJens Axboe } 220436885758c1b859392b80c616c338f9cfe9a9dfbJens Axboe if (!sd->vmsplice_to_user) 221f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe ret = fio_splice_read_old(td, io_u); 222f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe } else if (io_u->ddir == DDIR_WRITE) 2232866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe ret = fio_splice_write(td, io_u); 224a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe else if (io_u->ddir == DDIR_TRIM) 225a5f3027cb0495dfe217b2626d248fcc054e7e878Jens Axboe ret = do_io_u_trim(td, io_u); 22687dc1ab1b4df7b977f60e3d43533a896e2ee665bJens Axboe else 227f011531e61ae750cdf82074e0dea1379b07fa239Jens Axboe ret = do_io_u_sync(td, io_u); 2282866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 229cec6b55da1c282b5b91ad346c7804171fccf151eJens Axboe if (ret != (int) io_u->xfer_buflen) { 23022819ec237297fc39435ed566bee01a4225bfb39Jens Axboe if (ret >= 0) { 231cec6b55da1c282b5b91ad346c7804171fccf151eJens Axboe io_u->resid = io_u->xfer_buflen - ret; 232cec6b55da1c282b5b91ad346c7804171fccf151eJens Axboe io_u->error = 0; 23336167d82e5f49dee91c6d2cd426068edee90e36fJens Axboe return FIO_Q_COMPLETED; 2342866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe } else 2352866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe io_u->error = errno; 2362866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe } 2372866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2380aa417a9515f9dc17523c1870f6409370e94ca19Jens Axboe if (io_u->error) { 239e1161c325f7866bae879e686d1c673ca32ab09aeJens Axboe td_verror(td, io_u->error, "xfer"); 2400aa417a9515f9dc17523c1870f6409370e94ca19Jens Axboe if (io_u->error == EINVAL) 2410aa417a9515f9dc17523c1870f6409370e94ca19Jens Axboe log_err("fio: looks like splice doesn't work on this" 2420aa417a9515f9dc17523c1870f6409370e94ca19Jens Axboe " file system\n"); 2430aa417a9515f9dc17523c1870f6409370e94ca19Jens Axboe } 2442866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 24536167d82e5f49dee91c6d2cd426068edee90e36fJens Axboe return FIO_Q_COMPLETED; 2462866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 2472866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2482866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic void fio_spliceio_cleanup(struct thread_data *td) 2492866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 250eda3a60699e1d96bb68875ef2169ca819eb8f4f9Elliott Hughes struct spliceio_data *sd = td->io_ops_data; 2512866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2522866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe if (sd) { 2532866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe close(sd->pipe[0]); 2542866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe close(sd->pipe[1]); 2552866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe free(sd); 2562866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe } 2572866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 2582866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2592866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboestatic int fio_spliceio_init(struct thread_data *td) 2602866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe{ 2612866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe struct spliceio_data *sd = malloc(sizeof(*sd)); 2622866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2632866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe if (pipe(sd->pipe) < 0) { 264e1161c325f7866bae879e686d1c673ca32ab09aeJens Axboe td_verror(td, errno, "pipe"); 2652866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe free(sd); 2662866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe return 1; 2672866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe } 2682866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 269f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe /* 270436885758c1b859392b80c616c338f9cfe9a9dfbJens Axboe * Assume this work, we'll reset this if it doesn't 271f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe */ 272436885758c1b859392b80c616c338f9cfe9a9dfbJens Axboe sd->vmsplice_to_user = 1; 273f24254e1d142e80dc76d1f2e316da12f6ec125eeJens Axboe 27481887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe /* 2758b850243258107412710cd38252f0cfb8802593dJens Axboe * Works with "real" vmsplice to user, eg mapping pages directly. 2768b850243258107412710cd38252f0cfb8802593dJens Axboe * Reset if we fail. 2778b850243258107412710cd38252f0cfb8802593dJens Axboe */ 2788b850243258107412710cd38252f0cfb8802593dJens Axboe sd->vmsplice_to_user_map = 1; 2798b850243258107412710cd38252f0cfb8802593dJens Axboe 2808b850243258107412710cd38252f0cfb8802593dJens Axboe /* 28181887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe * And if vmsplice_to_user works, we definitely need aligned 28281887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe * buffers. Just set ->odirect to force that. 28381887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe */ 28481887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe if (td_read(td)) 28576a3179ec3b849e4d4b0324ab3f517beec816bdfJens Axboe td->o.mem_align = 1; 28681887d5dcd608492e59ed3fb2ac8fcb463223120Jens Axboe 287eda3a60699e1d96bb68875ef2169ca819eb8f4f9Elliott Hughes td->io_ops_data = sd; 2882866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe return 0; 2892866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe} 2902866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe 2915f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboestatic struct ioengine_ops ioengine = { 2922866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe .name = "splice", 2932866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe .version = FIO_IOOPS_VERSION, 2942866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe .init = fio_spliceio_init, 2952866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe .queue = fio_spliceio_queue, 2962866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe .cleanup = fio_spliceio_cleanup, 297b5af82930ccfd7dda6a1b11794efb452eb76d8dcJens Axboe .open_file = generic_open_file, 298b5af82930ccfd7dda6a1b11794efb452eb76d8dcJens Axboe .close_file = generic_close_file, 299df9c26b10275a631e83e7cc92d5f7384998b2c49Jens Axboe .get_file_size = generic_get_file_size, 3009c0d224129b0c59698e4c77e7fed00dc8cbb50e1Jens Axboe .flags = FIO_SYNCIO | FIO_PIPEIO, 3012866c82d598e30604d8a92723c664ee6ced90fb0Jens Axboe}; 30234cfcdafa994a0a75120e498c51eda08bde5df72Jens Axboe 3035f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboestatic void fio_init fio_spliceio_register(void) 3045f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe{ 3055f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe register_ioengine(&ioengine); 3065f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe} 3075f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe 3085f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboestatic void fio_exit fio_spliceio_unregister(void) 3095f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe{ 3105f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe unregister_ioengine(&ioengine); 3115f350952eff89948bfbf1eb6ac4d3d08a9109581Jens Axboe} 312