splice.c revision 436885758c1b859392b80c616c338f9cfe9a9dfb
1/* 2 * splice engine 3 * 4 * IO engine that transfers data by doing splices to/from pipes and 5 * the files. 6 * 7 */ 8#include <stdio.h> 9#include <stdlib.h> 10#include <unistd.h> 11#include <errno.h> 12#include <assert.h> 13#include <sys/poll.h> 14 15#include "../fio.h" 16 17#ifdef FIO_HAVE_SPLICE 18 19struct spliceio_data { 20 int pipe[2]; 21 int vmsplice_to_user; 22}; 23 24/* 25 * vmsplice didn't use to support splicing to user space, this is the old 26 * variant of getting that job done. Doesn't make a lot of sense, but it 27 * uses splices to move data from the source into a pipe. 28 */ 29static int fio_splice_read_old(struct thread_data *td, struct io_u *io_u) 30{ 31 struct spliceio_data *sd = td->io_ops->data; 32 struct fio_file *f = io_u->file; 33 int ret, ret2, buflen; 34 off_t offset; 35 void *p; 36 37 offset = io_u->offset; 38 buflen = io_u->xfer_buflen; 39 p = io_u->xfer_buf; 40 while (buflen) { 41 int this_len = buflen; 42 43 if (this_len > SPLICE_DEF_SIZE) 44 this_len = SPLICE_DEF_SIZE; 45 46 ret = splice(f->fd, &offset, sd->pipe[1], NULL, this_len, SPLICE_F_MORE); 47 if (ret < 0) { 48 if (errno == ENODATA || errno == EAGAIN) 49 continue; 50 51 return -errno; 52 } 53 54 buflen -= ret; 55 56 while (ret) { 57 ret2 = read(sd->pipe[0], p, ret); 58 if (ret2 < 0) 59 return -errno; 60 61 ret -= ret2; 62 p += ret2; 63 } 64 } 65 66 return io_u->xfer_buflen; 67} 68 69/* 70 * We can now vmsplice into userspace, so do the transfer by splicing into 71 * a pipe and vmsplicing that into userspace. 72 */ 73static int fio_splice_read(struct thread_data *td, struct io_u *io_u) 74{ 75 struct spliceio_data *sd = td->io_ops->data; 76 struct fio_file *f = io_u->file; 77 struct iovec iov; 78 int ret, buflen; 79 off_t offset; 80 void *p; 81 82 offset = io_u->offset; 83 buflen = io_u->xfer_buflen; 84 p = io_u->xfer_buf; 85 while (buflen) { 86 int this_len = buflen; 87 88 if (this_len > SPLICE_DEF_SIZE) 89 this_len = SPLICE_DEF_SIZE; 90 91 ret = splice(f->fd, &offset, sd->pipe[1], NULL, this_len, SPLICE_F_MORE); 92 if (ret < 0) { 93 if (errno == ENODATA || errno == EAGAIN) 94 continue; 95 96 return -errno; 97 } 98 99 buflen -= ret; 100 iov.iov_base = p; 101 iov.iov_len = ret; 102 p += ret; 103 104 while (iov.iov_len) { 105 ret = vmsplice(sd->pipe[0], &iov, 1, SPLICE_F_MOVE); 106 if (ret < 0) 107 return -errno; 108 else if (!ret) 109 return -ENODATA; 110 111 iov.iov_len -= ret; 112 iov.iov_base += ret; 113 } 114 } 115 116 return io_u->xfer_buflen; 117} 118 119/* 120 * For splice writing, we can vmsplice our data buffer directly into a 121 * pipe and then splice that to a file. 122 */ 123static int fio_splice_write(struct thread_data *td, struct io_u *io_u) 124{ 125 struct spliceio_data *sd = td->io_ops->data; 126 struct iovec iov = { 127 .iov_base = io_u->xfer_buf, 128 .iov_len = io_u->xfer_buflen, 129 }; 130 struct pollfd pfd = { .fd = sd->pipe[1], .events = POLLOUT, }; 131 struct fio_file *f = io_u->file; 132 off_t off = io_u->offset; 133 int ret, ret2; 134 135 while (iov.iov_len) { 136 if (poll(&pfd, 1, -1) < 0) 137 return errno; 138 139 ret = vmsplice(sd->pipe[1], &iov, 1, SPLICE_F_NONBLOCK); 140 if (ret < 0) 141 return -errno; 142 143 iov.iov_len -= ret; 144 iov.iov_base += ret; 145 146 while (ret) { 147 ret2 = splice(sd->pipe[0], NULL, f->fd, &off, ret, 0); 148 if (ret2 < 0) 149 return -errno; 150 151 ret -= ret2; 152 } 153 } 154 155 return io_u->xfer_buflen; 156} 157 158static int fio_spliceio_queue(struct thread_data *td, struct io_u *io_u) 159{ 160 struct spliceio_data *sd = td->io_ops->data; 161 int ret; 162 163 if (io_u->ddir == DDIR_READ) { 164 if (sd->vmsplice_to_user) { 165 ret = fio_splice_read(td, io_u); 166 /* 167 * This kernel doesn't support vmsplice to user 168 * space. Reset the vmsplice_to_user flag, so that 169 * we retry below and don't hit this path again. 170 */ 171 if (ret == -EBADF) 172 sd->vmsplice_to_user = 0; 173 } 174 if (!sd->vmsplice_to_user) 175 ret = fio_splice_read_old(td, io_u); 176 } else if (io_u->ddir == DDIR_WRITE) 177 ret = fio_splice_write(td, io_u); 178 else 179 ret = fsync(io_u->file->fd); 180 181 if (ret != (int) io_u->xfer_buflen) { 182 if (ret >= 0) { 183 io_u->resid = io_u->xfer_buflen - ret; 184 io_u->error = 0; 185 return FIO_Q_COMPLETED; 186 } else 187 io_u->error = errno; 188 } 189 190 if (io_u->error) 191 td_verror(td, io_u->error, "xfer"); 192 193 return FIO_Q_COMPLETED; 194} 195 196static void fio_spliceio_cleanup(struct thread_data *td) 197{ 198 struct spliceio_data *sd = td->io_ops->data; 199 200 if (sd) { 201 close(sd->pipe[0]); 202 close(sd->pipe[1]); 203 free(sd); 204 td->io_ops->data = NULL; 205 } 206} 207 208static int fio_spliceio_init(struct thread_data *td) 209{ 210 struct spliceio_data *sd = malloc(sizeof(*sd)); 211 212 if (pipe(sd->pipe) < 0) { 213 td_verror(td, errno, "pipe"); 214 free(sd); 215 return 1; 216 } 217 218 /* 219 * Assume this work, we'll reset this if it doesn't 220 */ 221 sd->vmsplice_to_user = 1; 222 223 td->io_ops->data = sd; 224 return 0; 225} 226 227static struct ioengine_ops ioengine = { 228 .name = "splice", 229 .version = FIO_IOOPS_VERSION, 230 .init = fio_spliceio_init, 231 .queue = fio_spliceio_queue, 232 .cleanup = fio_spliceio_cleanup, 233 .open_file = generic_open_file, 234 .close_file = generic_close_file, 235 .flags = FIO_SYNCIO, 236}; 237 238#else /* FIO_HAVE_SPLICE */ 239 240/* 241 * When we have a proper configure system in place, we simply wont build 242 * and install this io engine. For now install a crippled version that 243 * just complains and fails to load. 244 */ 245static int fio_spliceio_init(struct thread_data fio_unused *td) 246{ 247 fprintf(stderr, "fio: splice not available\n"); 248 return 1; 249} 250 251static struct ioengine_ops ioengine = { 252 .name = "splice", 253 .version = FIO_IOOPS_VERSION, 254 .init = fio_spliceio_init, 255}; 256 257#endif 258 259static void fio_init fio_spliceio_register(void) 260{ 261 register_ioengine(&ioengine); 262} 263 264static void fio_exit fio_spliceio_unregister(void) 265{ 266 unregister_ioengine(&ioengine); 267} 268