vhost.h revision c4fcb586c337f8b8de3a3d7ba8514eed03695f6e
1#ifndef _VHOST_H 2#define _VHOST_H 3 4#include <linux/eventfd.h> 5#include <linux/vhost.h> 6#include <linux/mm.h> 7#include <linux/mutex.h> 8#include <linux/poll.h> 9#include <linux/file.h> 10#include <linux/skbuff.h> 11#include <linux/uio.h> 12#include <linux/virtio_config.h> 13#include <linux/virtio_ring.h> 14#include <linux/atomic.h> 15 16/* 17 * For transmit, used buffer len is unused; we override it to track buffer 18 * status internally; used for zerocopy tx only. 19 */ 20/* Lower device DMA failed */ 21#define VHOST_DMA_FAILED_LEN 3 22/* Lower device DMA done */ 23#define VHOST_DMA_DONE_LEN 2 24/* Lower device DMA in progress */ 25#define VHOST_DMA_IN_PROGRESS 1 26/* Buffer unused */ 27#define VHOST_DMA_CLEAR_LEN 0 28 29#define VHOST_DMA_IS_DONE(len) ((len) >= VHOST_DMA_DONE_LEN) 30 31struct vhost_device; 32 33struct vhost_work; 34typedef void (*vhost_work_fn_t)(struct vhost_work *work); 35 36struct vhost_work { 37 struct list_head node; 38 vhost_work_fn_t fn; 39 wait_queue_head_t done; 40 int flushing; 41 unsigned queue_seq; 42 unsigned done_seq; 43}; 44 45/* Poll a file (eventfd or socket) */ 46/* Note: there's nothing vhost specific about this structure. */ 47struct vhost_poll { 48 poll_table table; 49 wait_queue_head_t *wqh; 50 wait_queue_t wait; 51 struct vhost_work work; 52 unsigned long mask; 53 struct vhost_dev *dev; 54}; 55 56void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn); 57void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work); 58 59void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn, 60 unsigned long mask, struct vhost_dev *dev); 61void vhost_poll_start(struct vhost_poll *poll, struct file *file); 62void vhost_poll_stop(struct vhost_poll *poll); 63void vhost_poll_flush(struct vhost_poll *poll); 64void vhost_poll_queue(struct vhost_poll *poll); 65 66struct vhost_log { 67 u64 addr; 68 u64 len; 69}; 70 71struct vhost_virtqueue; 72 73struct vhost_ubuf_ref { 74 struct kref kref; 75 wait_queue_head_t wait; 76 struct vhost_virtqueue *vq; 77}; 78 79struct vhost_ubuf_ref *vhost_ubuf_alloc(struct vhost_virtqueue *, bool zcopy); 80void vhost_ubuf_put(struct vhost_ubuf_ref *); 81void vhost_ubuf_put_and_wait(struct vhost_ubuf_ref *); 82 83/* The virtqueue structure describes a queue attached to a device. */ 84struct vhost_virtqueue { 85 struct vhost_dev *dev; 86 87 /* The actual ring of buffers. */ 88 struct mutex mutex; 89 unsigned int num; 90 struct vring_desc __user *desc; 91 struct vring_avail __user *avail; 92 struct vring_used __user *used; 93 struct file *kick; 94 struct file *call; 95 struct file *error; 96 struct eventfd_ctx *call_ctx; 97 struct eventfd_ctx *error_ctx; 98 struct eventfd_ctx *log_ctx; 99 100 struct vhost_poll poll; 101 102 /* The routine to call when the Guest pings us, or timeout. */ 103 vhost_work_fn_t handle_kick; 104 105 /* Last available index we saw. */ 106 u16 last_avail_idx; 107 108 /* Caches available index value from user. */ 109 u16 avail_idx; 110 111 /* Last index we used. */ 112 u16 last_used_idx; 113 114 /* Used flags */ 115 u16 used_flags; 116 117 /* Last used index value we have signalled on */ 118 u16 signalled_used; 119 120 /* Last used index value we have signalled on */ 121 bool signalled_used_valid; 122 123 /* Log writes to used structure. */ 124 bool log_used; 125 u64 log_addr; 126 127 struct iovec iov[UIO_MAXIOV]; 128 /* hdr is used to store the virtio header. 129 * Since each iovec has >= 1 byte length, we never need more than 130 * header length entries to store the header. */ 131 struct iovec hdr[sizeof(struct virtio_net_hdr_mrg_rxbuf)]; 132 struct iovec *indirect; 133 size_t vhost_hlen; 134 size_t sock_hlen; 135 struct vring_used_elem *heads; 136 /* We use a kind of RCU to access private pointer. 137 * All readers access it from worker, which makes it possible to 138 * flush the vhost_work instead of synchronize_rcu. Therefore readers do 139 * not need to call rcu_read_lock/rcu_read_unlock: the beginning of 140 * vhost_work execution acts instead of rcu_read_lock() and the end of 141 * vhost_work execution acts instead of rcu_read_unlock(). 142 * Writers use virtqueue mutex. */ 143 void __rcu *private_data; 144 /* Log write descriptors */ 145 void __user *log_base; 146 struct vhost_log *log; 147 /* vhost zerocopy support fields below: */ 148 /* last used idx for outstanding DMA zerocopy buffers */ 149 int upend_idx; 150 /* first used idx for DMA done zerocopy buffers */ 151 int done_idx; 152 /* an array of userspace buffers info */ 153 struct ubuf_info *ubuf_info; 154 /* Reference counting for outstanding ubufs. 155 * Protected by vq mutex. Writers must also take device mutex. */ 156 struct vhost_ubuf_ref *ubufs; 157}; 158 159struct vhost_dev { 160 /* Readers use RCU to access memory table pointer 161 * log base pointer and features. 162 * Writers use mutex below.*/ 163 struct vhost_memory __rcu *memory; 164 struct mm_struct *mm; 165 struct mutex mutex; 166 unsigned acked_features; 167 struct vhost_virtqueue *vqs; 168 int nvqs; 169 struct file *log_file; 170 struct eventfd_ctx *log_ctx; 171 spinlock_t work_lock; 172 struct list_head work_list; 173 struct task_struct *worker; 174}; 175 176long vhost_dev_init(struct vhost_dev *, struct vhost_virtqueue *vqs, int nvqs); 177long vhost_dev_check_owner(struct vhost_dev *); 178long vhost_dev_reset_owner(struct vhost_dev *); 179void vhost_dev_cleanup(struct vhost_dev *, bool locked); 180long vhost_dev_ioctl(struct vhost_dev *, unsigned int ioctl, unsigned long arg); 181int vhost_vq_access_ok(struct vhost_virtqueue *vq); 182int vhost_log_access_ok(struct vhost_dev *); 183 184int vhost_get_vq_desc(struct vhost_dev *, struct vhost_virtqueue *, 185 struct iovec iov[], unsigned int iov_count, 186 unsigned int *out_num, unsigned int *in_num, 187 struct vhost_log *log, unsigned int *log_num); 188void vhost_discard_vq_desc(struct vhost_virtqueue *, int n); 189 190int vhost_init_used(struct vhost_virtqueue *); 191int vhost_add_used(struct vhost_virtqueue *, unsigned int head, int len); 192int vhost_add_used_n(struct vhost_virtqueue *, struct vring_used_elem *heads, 193 unsigned count); 194void vhost_add_used_and_signal(struct vhost_dev *, struct vhost_virtqueue *, 195 unsigned int id, int len); 196void vhost_add_used_and_signal_n(struct vhost_dev *, struct vhost_virtqueue *, 197 struct vring_used_elem *heads, unsigned count); 198void vhost_signal(struct vhost_dev *, struct vhost_virtqueue *); 199void vhost_disable_notify(struct vhost_dev *, struct vhost_virtqueue *); 200bool vhost_enable_notify(struct vhost_dev *, struct vhost_virtqueue *); 201 202int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log, 203 unsigned int log_num, u64 len); 204void vhost_zerocopy_callback(struct ubuf_info *, bool); 205int vhost_zerocopy_signal_used(struct vhost_virtqueue *vq); 206 207#define vq_err(vq, fmt, ...) do { \ 208 pr_debug(pr_fmt(fmt), ##__VA_ARGS__); \ 209 if ((vq)->error_ctx) \ 210 eventfd_signal((vq)->error_ctx, 1);\ 211 } while (0) 212 213enum { 214 VHOST_FEATURES = (1ULL << VIRTIO_F_NOTIFY_ON_EMPTY) | 215 (1ULL << VIRTIO_RING_F_INDIRECT_DESC) | 216 (1ULL << VIRTIO_RING_F_EVENT_IDX) | 217 (1ULL << VHOST_F_LOG_ALL), 218 VHOST_NET_FEATURES = VHOST_FEATURES | 219 (1ULL << VHOST_NET_F_VIRTIO_NET_HDR) | 220 (1ULL << VIRTIO_NET_F_MRG_RXBUF), 221}; 222 223static inline int vhost_has_feature(struct vhost_dev *dev, int bit) 224{ 225 unsigned acked_features; 226 227 /* TODO: check that we are running from vhost_worker or dev mutex is 228 * held? */ 229 acked_features = rcu_dereference_index_check(dev->acked_features, 1); 230 return acked_features & (1 << bit); 231} 232 233void vhost_enable_zcopy(int vq); 234 235#endif 236