1#ifndef FIO_IOENGINE_H
2#define FIO_IOENGINE_H
3
4#include "compiler/compiler.h"
5#include "os/os.h"
6#include "log.h"
7#include "io_ddir.h"
8#include "debug.h"
9#include "file.h"
10
11#ifdef CONFIG_LIBAIO
12#include <libaio.h>
13#endif
14#ifdef CONFIG_GUASI
15#include <guasi.h>
16#endif
17
18#define FIO_IOOPS_VERSION	18
19
20enum {
21	IO_U_F_FREE		= 1 << 0,
22	IO_U_F_FLIGHT		= 1 << 1,
23	IO_U_F_FREE_DEF		= 1 << 2,
24	IO_U_F_IN_CUR_DEPTH	= 1 << 3,
25	IO_U_F_BUSY_OK		= 1 << 4,
26	IO_U_F_TRIMMED		= 1 << 5,
27	IO_U_F_BARRIER		= 1 << 6,
28	IO_U_F_VER_LIST		= 1 << 7,
29};
30
31/*
32 * The io unit
33 */
34struct io_u {
35	struct timeval start_time;
36	struct timeval issue_time;
37
38	struct fio_file *file;
39	unsigned int flags;
40	enum fio_ddir ddir;
41
42	/*
43	 * For replay workloads, we may want to account as a different
44	 * IO type than what is being submitted.
45	 */
46	enum fio_ddir acct_ddir;
47
48	/*
49	 * Allocated/set buffer and length
50	 */
51	unsigned long buflen;
52	unsigned long long offset;
53	unsigned short numberio;
54	void *buf;
55
56	/*
57	 * Initial seed for generating the buffer contents
58	 */
59	uint64_t rand_seed;
60
61	/*
62	 * IO engine state, may be different from above when we get
63	 * partial transfers / residual data counts
64	 */
65	void *xfer_buf;
66	unsigned long xfer_buflen;
67
68	/*
69	 * Parameter related to pre-filled buffers and
70	 * their size to handle variable block sizes.
71	 */
72	unsigned long buf_filled_len;
73
74	struct io_piece *ipo;
75
76	unsigned int resid;
77	unsigned int error;
78
79	/*
80	 * io engine private data
81	 */
82	union {
83		unsigned int index;
84		unsigned int seen;
85		void *engine_data;
86	};
87
88	struct flist_head verify_list;
89
90	/*
91	 * Callback for io completion
92	 */
93	int (*end_io)(struct thread_data *, struct io_u *);
94
95	union {
96#ifdef CONFIG_LIBAIO
97		struct iocb iocb;
98#endif
99#ifdef CONFIG_POSIXAIO
100		os_aiocb_t aiocb;
101#endif
102#ifdef FIO_HAVE_SGIO
103		struct sg_io_hdr hdr;
104#endif
105#ifdef CONFIG_GUASI
106		guasi_req_t greq;
107#endif
108#ifdef CONFIG_SOLARISAIO
109		aio_result_t resultp;
110#endif
111#ifdef FIO_HAVE_BINJECT
112		struct b_user_cmd buc;
113#endif
114#ifdef CONFIG_RDMA
115		struct ibv_mr *mr;
116#endif
117		void *mmap_data;
118	};
119};
120
121/*
122 * io_ops->queue() return values
123 */
124enum {
125	FIO_Q_COMPLETED	= 0,		/* completed sync */
126	FIO_Q_QUEUED	= 1,		/* queued, will complete async */
127	FIO_Q_BUSY	= 2,		/* no more room, call ->commit() */
128};
129
130struct ioengine_ops {
131	struct flist_head list;
132	char name[16];
133	int version;
134	int flags;
135	int (*setup)(struct thread_data *);
136	int (*init)(struct thread_data *);
137	int (*prep)(struct thread_data *, struct io_u *);
138	int (*queue)(struct thread_data *, struct io_u *);
139	int (*commit)(struct thread_data *);
140	int (*getevents)(struct thread_data *, unsigned int, unsigned int, struct timespec *);
141	struct io_u *(*event)(struct thread_data *, int);
142	int (*cancel)(struct thread_data *, struct io_u *);
143	void (*cleanup)(struct thread_data *);
144	int (*open_file)(struct thread_data *, struct fio_file *);
145	int (*close_file)(struct thread_data *, struct fio_file *);
146	int (*get_file_size)(struct thread_data *, struct fio_file *);
147	void (*terminate)(struct thread_data *);
148	int (*io_u_init)(struct thread_data *, struct io_u *);
149	void (*io_u_free)(struct thread_data *, struct io_u *);
150	int option_struct_size;
151	struct fio_option *options;
152	void *data;
153	void *dlhandle;
154};
155
156enum fio_ioengine_flags {
157	FIO_SYNCIO	= 1 << 0,	/* io engine has synchronous ->queue */
158	FIO_RAWIO	= 1 << 1,	/* some sort of direct/raw io */
159	FIO_DISKLESSIO	= 1 << 2,	/* no disk involved */
160	FIO_NOEXTEND	= 1 << 3,	/* engine can't extend file */
161	FIO_NODISKUTIL  = 1 << 4,	/* diskutil can't handle filename */
162	FIO_UNIDIR	= 1 << 5,	/* engine is uni-directional */
163	FIO_NOIO	= 1 << 6,	/* thread does only pseudo IO */
164	FIO_PIPEIO	= 1 << 7,	/* input/output no seekable */
165	FIO_BARRIER	= 1 << 8,	/* engine supports barriers */
166	FIO_MEMALIGN	= 1 << 9,	/* engine wants aligned memory */
167	FIO_BIT_BASED	= 1 << 10,	/* engine uses a bit base (e.g. uses Kbit as opposed to KB) */
168};
169
170/*
171 * External engine defined symbol to fill in the engine ops structure
172 */
173typedef void (*get_ioengine_t)(struct ioengine_ops **);
174
175/*
176 * io engine entry points
177 */
178extern int __must_check td_io_init(struct thread_data *);
179extern int __must_check td_io_prep(struct thread_data *, struct io_u *);
180extern int __must_check td_io_queue(struct thread_data *, struct io_u *);
181extern int __must_check td_io_sync(struct thread_data *, struct fio_file *);
182extern int __must_check td_io_getevents(struct thread_data *, unsigned int, unsigned int, struct timespec *);
183extern int __must_check td_io_commit(struct thread_data *);
184extern int __must_check td_io_open_file(struct thread_data *, struct fio_file *);
185extern int td_io_close_file(struct thread_data *, struct fio_file *);
186extern int __must_check td_io_get_file_size(struct thread_data *, struct fio_file *);
187
188extern struct ioengine_ops *load_ioengine(struct thread_data *, const char *);
189extern void register_ioengine(struct ioengine_ops *);
190extern void unregister_ioengine(struct ioengine_ops *);
191extern void free_ioengine(struct thread_data *);
192extern void close_ioengine(struct thread_data *);
193
194extern int fio_show_ioengine_help(const char *engine);
195
196/*
197 * io unit handling
198 */
199extern struct io_u *__get_io_u(struct thread_data *);
200extern struct io_u *get_io_u(struct thread_data *);
201extern void put_io_u(struct thread_data *, struct io_u *);
202extern void clear_io_u(struct thread_data *, struct io_u *);
203extern void requeue_io_u(struct thread_data *, struct io_u **);
204extern int __must_check io_u_sync_complete(struct thread_data *, struct io_u *, uint64_t *);
205extern int __must_check io_u_queued_complete(struct thread_data *, int, uint64_t *);
206extern void io_u_queued(struct thread_data *, struct io_u *);
207extern void io_u_quiesce(struct thread_data *);
208extern void io_u_log_error(struct thread_data *, struct io_u *);
209extern void io_u_mark_depth(struct thread_data *, unsigned int);
210extern void fill_io_buffer(struct thread_data *, void *, unsigned int, unsigned int);
211extern void io_u_fill_buffer(struct thread_data *td, struct io_u *, unsigned int, unsigned int);
212void io_u_mark_complete(struct thread_data *, unsigned int);
213void io_u_mark_submit(struct thread_data *, unsigned int);
214int queue_full(struct thread_data *);
215
216int do_io_u_sync(struct thread_data *, struct io_u *);
217int do_io_u_trim(struct thread_data *, struct io_u *);
218
219#ifdef FIO_INC_DEBUG
220static inline void dprint_io_u(struct io_u *io_u, const char *p)
221{
222	struct fio_file *f = io_u->file;
223
224	dprint(FD_IO, "%s: io_u %p: off=%llu/len=%lu/ddir=%d", p, io_u,
225					(unsigned long long) io_u->offset,
226					io_u->buflen, io_u->ddir);
227	if (fio_debug & (1 << FD_IO)) {
228		if (f)
229			log_info("/%s", f->file_name);
230
231		log_info("\n");
232	}
233}
234#else
235#define dprint_io_u(io_u, p)
236#endif
237
238static inline enum fio_ddir acct_ddir(struct io_u *io_u)
239{
240	if (io_u->acct_ddir != -1)
241		return io_u->acct_ddir;
242
243	return io_u->ddir;
244}
245
246#endif
247