virtio_ring.c revision 3b720b8c865098c49c1570b6b5c7832bcfa6e6c2
10a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell/* Virtio ring implementation. 20a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * 30a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * Copyright 2007 Rusty Russell IBM Corporation 40a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * 50a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * This program is free software; you can redistribute it and/or modify 60a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * it under the terms of the GNU General Public License as published by 70a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * the Free Software Foundation; either version 2 of the License, or 80a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * (at your option) any later version. 90a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * 100a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * This program is distributed in the hope that it will be useful, 110a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * but WITHOUT ANY WARRANTY; without even the implied warranty of 120a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 130a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * GNU General Public License for more details. 140a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * 150a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * You should have received a copy of the GNU General Public License 160a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * along with this program; if not, write to the Free Software 170a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 180a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell */ 190a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#include <linux/virtio.h> 200a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#include <linux/virtio_ring.h> 21e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell#include <linux/virtio_config.h> 220a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#include <linux/device.h> 235a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 24b5a2c4f1996d1d258ac10897e0bbb57c6904a735Paul Gortmaker#include <linux/module.h> 250a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 26d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin/* virtio guest is communicating with a virtual "device" that actually runs on 27d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin * a host processor. Memory barriers are used to control SMP effects. */ 28d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin#ifdef CONFIG_SMP 29d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin/* Where possible, use SMP barriers which are more lightweight than mandatory 30d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin * barriers, because mandatory barriers control MMIO effects on accesses 317b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell * through relaxed memory I/O windows (which virtio-pci does not use). */ 327b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell#define virtio_mb(vq) \ 337b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell do { if ((vq)->weak_barriers) smp_mb(); else mb(); } while(0) 347b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell#define virtio_rmb(vq) \ 357b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0) 367b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell#define virtio_wmb(vq) \ 377b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0) 38d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin#else 39d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin/* We must force memory ordering even if guest is UP since host could be 40d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin * running on another CPU, but SMP barriers are defined to barrier() in that 41d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin * configuration. So fall back to mandatory barriers instead. */ 427b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell#define virtio_mb(vq) mb() 437b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell#define virtio_rmb(vq) rmb() 447b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell#define virtio_wmb(vq) wmb() 45d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin#endif 46d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin 470a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#ifdef DEBUG 480a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell/* For development, we want to crash whenever the ring is screwed. */ 499499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell#define BAD_RING(_vq, fmt, args...) \ 509499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell do { \ 519499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell dev_err(&(_vq)->vq.vdev->dev, \ 529499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell "%s:"fmt, (_vq)->vq.name, ##args); \ 539499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell BUG(); \ 549499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell } while (0) 55c5f841f1780dad7efb7eca092f60742d47f47d25Rusty Russell/* Caller is supposed to guarantee no reentry. */ 56c5f841f1780dad7efb7eca092f60742d47f47d25Rusty Russell#define START_USE(_vq) \ 57c5f841f1780dad7efb7eca092f60742d47f47d25Rusty Russell do { \ 58c5f841f1780dad7efb7eca092f60742d47f47d25Rusty Russell if ((_vq)->in_use) \ 599499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell panic("%s:in_use = %i\n", \ 609499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell (_vq)->vq.name, (_vq)->in_use); \ 61c5f841f1780dad7efb7eca092f60742d47f47d25Rusty Russell (_vq)->in_use = __LINE__; \ 629499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell } while (0) 633a35ce7dcefe9e80a00603a195269fbaf6e7d901Roel Kluin#define END_USE(_vq) \ 6497a545ab6ce922a0f868d192718a48a0091ebc5eRusty Russell do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0) 650a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#else 669499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell#define BAD_RING(_vq, fmt, args...) \ 679499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell do { \ 689499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell dev_err(&_vq->vq.vdev->dev, \ 699499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell "%s:"fmt, (_vq)->vq.name, ##args); \ 709499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell (_vq)->broken = true; \ 719499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell } while (0) 720a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#define START_USE(vq) 730a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#define END_USE(vq) 740a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#endif 750a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 760a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russellstruct vring_virtqueue 770a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell{ 780a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell struct virtqueue vq; 790a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 800a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Actual memory layout for this queue */ 810a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell struct vring vring; 820a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 837b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell /* Can we use weak barriers? */ 847b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell bool weak_barriers; 857b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell 860a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Other side has made a mess, don't try any more. */ 870a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell bool broken; 880a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 899fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin /* Host supports indirect buffers */ 909fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin bool indirect; 919fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 92a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin /* Host publishes avail event idx */ 93a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin bool event; 94a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin 950a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Number of free buffers */ 960a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell unsigned int num_free; 970a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Head of free buffer list. */ 980a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell unsigned int free_head; 990a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Number we've added since last sync. */ 1000a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell unsigned int num_added; 1010a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 1020a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Last used index we've seen. */ 1031bc4953ed44454c7f53d0b609445d1534981ee75Anthony Liguori u16 last_used_idx; 1040a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 1050a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* How to notify other side. FIXME: commonalize hcalls! */ 1060a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell void (*notify)(struct virtqueue *vq); 1070a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 1080a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#ifdef DEBUG 1090a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* They're supposed to lock for us. */ 1100a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell unsigned int in_use; 1110a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#endif 1120a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 1130a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Tokens for callbacks. */ 1140a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell void *data[]; 1150a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell}; 1160a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 1170a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) 1180a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 1199fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin/* Set up an indirect table of descriptors and add it to the queue. */ 1209fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlinstatic int vring_add_indirect(struct vring_virtqueue *vq, 1219fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin struct scatterlist sg[], 1229fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin unsigned int out, 123bbd603efb4238cf78083c00f0a81adfa8994aa33Michael S. Tsirkin unsigned int in, 124bbd603efb4238cf78083c00f0a81adfa8994aa33Michael S. Tsirkin gfp_t gfp) 1259fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin{ 1269fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin struct vring_desc *desc; 1279fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin unsigned head; 1289fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin int i; 1299fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 130bbd603efb4238cf78083c00f0a81adfa8994aa33Michael S. Tsirkin desc = kmalloc((out + in) * sizeof(struct vring_desc), gfp); 1319fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin if (!desc) 132686d363786a53ed28ee875b84ef24e6d5126ef6fMichael S. Tsirkin return -ENOMEM; 1339fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 1349fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin /* Transfer entries from the sg list into the indirect page */ 1359fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin for (i = 0; i < out; i++) { 1369fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin desc[i].flags = VRING_DESC_F_NEXT; 1379fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin desc[i].addr = sg_phys(sg); 1389fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin desc[i].len = sg->length; 1399fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin desc[i].next = i+1; 1409fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin sg++; 1419fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin } 1429fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin for (; i < (out + in); i++) { 1439fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin desc[i].flags = VRING_DESC_F_NEXT|VRING_DESC_F_WRITE; 1449fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin desc[i].addr = sg_phys(sg); 1459fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin desc[i].len = sg->length; 1469fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin desc[i].next = i+1; 1479fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin sg++; 1489fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin } 1499fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 1509fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin /* Last one doesn't continue. */ 1519fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin desc[i-1].flags &= ~VRING_DESC_F_NEXT; 1529fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin desc[i-1].next = 0; 1539fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 1549fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin /* We're about to use a buffer */ 1559fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin vq->num_free--; 1569fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 1579fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin /* Use a single buffer which doesn't continue */ 1589fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin head = vq->free_head; 1599fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin vq->vring.desc[head].flags = VRING_DESC_F_INDIRECT; 1609fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin vq->vring.desc[head].addr = virt_to_phys(desc); 1619fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin vq->vring.desc[head].len = i * sizeof(struct vring_desc); 1629fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 1639fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin /* Update free pointer */ 1649fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin vq->free_head = vq->vring.desc[head].next; 1659fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 1669fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin return head; 1679fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin} 1689fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 1695dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell/** 170f96fde41f7f9af6cf20f6a1919f5d9670f84d574Rusty Russell * virtqueue_add_buf - expose buffer to other end 1715dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @vq: the struct virtqueue we're talking about. 1725dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @sg: the description of the buffer(s). 1735dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @out_num: the number of sg readable by other side 1745dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @in_num: the number of sg which are writable (after readable ones) 1755dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @data: the token identifying the buffer. 1765dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @gfp: how to do memory allocations (if necessary). 1775dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 1785dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * Caller must ensure we don't call this with other virtqueue operations 1795dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * at the same time (except where noted). 1805dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 1815dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * Returns remaining capacity of queue or a negative error 1825dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * (ie. ENOSPC). Note that it only really makes sense to treat all 1835dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * positive return values as "available": indirect buffers mean that 1845dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * we can put an entire sg[] array inside a single queue entry. 1855dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell */ 186f96fde41f7f9af6cf20f6a1919f5d9670f84d574Rusty Russellint virtqueue_add_buf(struct virtqueue *_vq, 187f96fde41f7f9af6cf20f6a1919f5d9670f84d574Rusty Russell struct scatterlist sg[], 188f96fde41f7f9af6cf20f6a1919f5d9670f84d574Rusty Russell unsigned int out, 189f96fde41f7f9af6cf20f6a1919f5d9670f84d574Rusty Russell unsigned int in, 190f96fde41f7f9af6cf20f6a1919f5d9670f84d574Rusty Russell void *data, 191f96fde41f7f9af6cf20f6a1919f5d9670f84d574Rusty Russell gfp_t gfp) 1920a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell{ 1930a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell struct vring_virtqueue *vq = to_vvq(_vq); 1941fe9b6fef11771461e69ecd1bc8935a1c7c90cb5Michael S. Tsirkin unsigned int i, avail, uninitialized_var(prev); 1951fe9b6fef11771461e69ecd1bc8935a1c7c90cb5Michael S. Tsirkin int head; 1960a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 1979fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin START_USE(vq); 1989fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 1990a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell BUG_ON(data == NULL); 2009fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 2019fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin /* If the host supports indirect descriptor tables, and we have multiple 2029fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin * buffers, then go indirect. FIXME: tune this threshold */ 2039fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin if (vq->indirect && (out + in) > 1 && vq->num_free) { 204bbd603efb4238cf78083c00f0a81adfa8994aa33Michael S. Tsirkin head = vring_add_indirect(vq, sg, out, in, gfp); 2051fe9b6fef11771461e69ecd1bc8935a1c7c90cb5Michael S. Tsirkin if (likely(head >= 0)) 2069fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin goto add_head; 2079fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin } 2089fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 2090a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell BUG_ON(out + in > vq->vring.num); 2100a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell BUG_ON(out + in == 0); 2110a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 2120a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell if (vq->num_free < out + in) { 2130a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell pr_debug("Can't add buf len %i - avail = %i\n", 2140a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell out + in, vq->num_free); 21544653eae1407f79dff6f52fcf594ae84cb165ec4Rusty Russell /* FIXME: for historical reasons, we force a notify here if 21644653eae1407f79dff6f52fcf594ae84cb165ec4Rusty Russell * there are outgoing parts to the buffer. Presumably the 21744653eae1407f79dff6f52fcf594ae84cb165ec4Rusty Russell * host should service the ring ASAP. */ 21844653eae1407f79dff6f52fcf594ae84cb165ec4Rusty Russell if (out) 21944653eae1407f79dff6f52fcf594ae84cb165ec4Rusty Russell vq->notify(&vq->vq); 2200a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell END_USE(vq); 2210a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return -ENOSPC; 2220a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell } 2230a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 2240a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* We're about to use some buffers from the free list. */ 2250a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->num_free -= out + in; 2260a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 2270a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell head = vq->free_head; 2280a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell for (i = vq->free_head; out; i = vq->vring.desc[i].next, out--) { 2290a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vring.desc[i].flags = VRING_DESC_F_NEXT; 23015f9c8903cbdb02aee0f1bcf86a97c2e238b9a3dRusty Russell vq->vring.desc[i].addr = sg_phys(sg); 2310a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vring.desc[i].len = sg->length; 2320a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell prev = i; 2330a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell sg++; 2340a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell } 2350a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell for (; in; i = vq->vring.desc[i].next, in--) { 2360a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vring.desc[i].flags = VRING_DESC_F_NEXT|VRING_DESC_F_WRITE; 23715f9c8903cbdb02aee0f1bcf86a97c2e238b9a3dRusty Russell vq->vring.desc[i].addr = sg_phys(sg); 2380a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vring.desc[i].len = sg->length; 2390a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell prev = i; 2400a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell sg++; 2410a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell } 2420a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Last one doesn't continue. */ 2430a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vring.desc[prev].flags &= ~VRING_DESC_F_NEXT; 2440a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 2450a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Update free pointer */ 2460a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->free_head = i; 2470a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 2489fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlinadd_head: 2490a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Set token. */ 2500a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->data[head] = data; 2510a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 2520a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Put entry in available array (but don't update avail->idx until they 2533b720b8c865098c49c1570b6b5c7832bcfa6e6c2Rusty Russell * do sync). */ 2543b720b8c865098c49c1570b6b5c7832bcfa6e6c2Rusty Russell avail = ((vq->vring.avail->idx + vq->num_added++) & (vq->vring.num-1)); 2550a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vring.avail->ring[avail] = head; 2560a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 2570a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell pr_debug("Added buffer head %i to %p\n", head, vq); 2580a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell END_USE(vq); 2593c1b27d5043086a485f8526353ae9fe37bfa1065Rusty Russell 2603c1b27d5043086a485f8526353ae9fe37bfa1065Rusty Russell return vq->num_free; 2610a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell} 262f96fde41f7f9af6cf20f6a1919f5d9670f84d574Rusty RussellEXPORT_SYMBOL_GPL(virtqueue_add_buf); 2630a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 2645dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell/** 26541f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell * virtqueue_kick_prepare - first half of split virtqueue_kick call. 2665dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @vq: the struct virtqueue 2675dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 26841f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell * Instead of virtqueue_kick(), you can do: 26941f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell * if (virtqueue_kick_prepare(vq)) 27041f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell * virtqueue_notify(vq); 2715dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 27241f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell * This is sometimes useful because the virtqueue_kick_prepare() needs 27341f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell * to be serialized, but the actual virtqueue_notify() call does not. 2745dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell */ 27541f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russellbool virtqueue_kick_prepare(struct virtqueue *_vq) 2760a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell{ 2770a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell struct vring_virtqueue *vq = to_vvq(_vq); 278a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin u16 new, old; 27941f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell bool needs_kick; 28041f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell 2810a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell START_USE(vq); 2820a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Descriptors and available array need to be set before we expose the 2830a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * new available array entries. */ 2847b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell virtio_wmb(vq); 2850a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 286a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin old = vq->vring.avail->idx; 287a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin new = vq->vring.avail->idx = old + vq->num_added; 2880a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->num_added = 0; 2890a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 2900a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Need to update avail index before checking if we should notify */ 2917b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell virtio_mb(vq); 2920a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 29341f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell if (vq->event) { 29441f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell needs_kick = vring_need_event(vring_avail_event(&vq->vring), 29541f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell new, old); 29641f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell } else { 29741f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell needs_kick = !(vq->vring.used->flags & VRING_USED_F_NO_NOTIFY); 29841f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell } 2990a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell END_USE(vq); 30041f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell return needs_kick; 30141f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell} 30241f0377f73039ca6fe97a469d1941a89cd9757f1Rusty RussellEXPORT_SYMBOL_GPL(virtqueue_kick_prepare); 30341f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell 30441f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell/** 30541f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell * virtqueue_notify - second half of split virtqueue_kick call. 30641f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell * @vq: the struct virtqueue 30741f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell * 30841f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell * This does not need to be serialized. 30941f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell */ 31041f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russellvoid virtqueue_notify(struct virtqueue *_vq) 31141f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell{ 31241f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell struct vring_virtqueue *vq = to_vvq(_vq); 31341f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell 31441f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell /* Prod other side to tell it about changes. */ 31541f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell vq->notify(_vq); 31641f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell} 31741f0377f73039ca6fe97a469d1941a89cd9757f1Rusty RussellEXPORT_SYMBOL_GPL(virtqueue_notify); 31841f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell 31941f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell/** 32041f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell * virtqueue_kick - update after add_buf 32141f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell * @vq: the struct virtqueue 32241f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell * 32341f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell * After one or more virtqueue_add_buf calls, invoke this to kick 32441f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell * the other side. 32541f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell * 32641f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell * Caller must ensure we don't call this with other virtqueue 32741f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell * operations at the same time (except where noted). 32841f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell */ 32941f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russellvoid virtqueue_kick(struct virtqueue *vq) 33041f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell{ 33141f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell if (virtqueue_kick_prepare(vq)) 33241f0377f73039ca6fe97a469d1941a89cd9757f1Rusty Russell virtqueue_notify(vq); 3330a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell} 3347c5e9ed0c84e7d70d887878574590638d5572659Michael S. TsirkinEXPORT_SYMBOL_GPL(virtqueue_kick); 3350a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 3360a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russellstatic void detach_buf(struct vring_virtqueue *vq, unsigned int head) 3370a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell{ 3380a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell unsigned int i; 3390a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 3400a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Clear data ptr. */ 3410a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->data[head] = NULL; 3420a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 3430a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Put back on free list: find end */ 3440a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell i = head; 3459fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 3469fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin /* Free the indirect table */ 3479fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin if (vq->vring.desc[i].flags & VRING_DESC_F_INDIRECT) 3489fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin kfree(phys_to_virt(vq->vring.desc[i].addr)); 3499fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 3500a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell while (vq->vring.desc[i].flags & VRING_DESC_F_NEXT) { 3510a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell i = vq->vring.desc[i].next; 3520a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->num_free++; 3530a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell } 3540a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 3550a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vring.desc[i].next = vq->free_head; 3560a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->free_head = head; 3570a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Plus final descriptor */ 3580a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->num_free++; 3590a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell} 3600a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 3610a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russellstatic inline bool more_used(const struct vring_virtqueue *vq) 3620a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell{ 3630a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return vq->last_used_idx != vq->vring.used->idx; 3640a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell} 3650a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 3665dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell/** 3675dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * virtqueue_get_buf - get the next used buffer 3685dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @vq: the struct virtqueue we're talking about. 3695dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @len: the length written into the buffer 3705dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 3715dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * If the driver wrote data into the buffer, @len will be set to the 3725dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * amount written. This means you don't need to clear the buffer 3735dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * beforehand to ensure there's no data leakage in the case of short 3745dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * writes. 3755dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 3765dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * Caller must ensure we don't call this with other virtqueue 3775dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * operations at the same time (except where noted). 3785dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 3795dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * Returns NULL if there are no used buffers, or the "data" token 380f96fde41f7f9af6cf20f6a1919f5d9670f84d574Rusty Russell * handed to virtqueue_add_buf(). 3815dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell */ 3827c5e9ed0c84e7d70d887878574590638d5572659Michael S. Tsirkinvoid *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len) 3830a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell{ 3840a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell struct vring_virtqueue *vq = to_vvq(_vq); 3850a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell void *ret; 3860a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell unsigned int i; 3873b720b8c865098c49c1570b6b5c7832bcfa6e6c2Rusty Russell u16 last_used; 3880a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 3890a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell START_USE(vq); 3900a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 3915ef827526fc01820a7a80827802e9fad3f34f937Rusty Russell if (unlikely(vq->broken)) { 3925ef827526fc01820a7a80827802e9fad3f34f937Rusty Russell END_USE(vq); 3935ef827526fc01820a7a80827802e9fad3f34f937Rusty Russell return NULL; 3945ef827526fc01820a7a80827802e9fad3f34f937Rusty Russell } 3955ef827526fc01820a7a80827802e9fad3f34f937Rusty Russell 3960a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell if (!more_used(vq)) { 3970a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell pr_debug("No more buffers in queue\n"); 3980a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell END_USE(vq); 3990a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return NULL; 4000a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell } 4010a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 4022d61ba95034f1abbdec7729d52c740870a5eddb6Michael S. Tsirkin /* Only get used array entries after they have been exposed by host. */ 4037b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell virtio_rmb(vq); 4042d61ba95034f1abbdec7729d52c740870a5eddb6Michael S. Tsirkin 4053b720b8c865098c49c1570b6b5c7832bcfa6e6c2Rusty Russell last_used = (vq->last_used_idx & (vq->vring.num - 1)); 4063b720b8c865098c49c1570b6b5c7832bcfa6e6c2Rusty Russell i = vq->vring.used->ring[last_used].id; 4073b720b8c865098c49c1570b6b5c7832bcfa6e6c2Rusty Russell *len = vq->vring.used->ring[last_used].len; 4080a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 4090a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell if (unlikely(i >= vq->vring.num)) { 4100a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell BAD_RING(vq, "id %u out of range\n", i); 4110a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return NULL; 4120a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell } 4130a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell if (unlikely(!vq->data[i])) { 4140a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell BAD_RING(vq, "id %u is not a head!\n", i); 4150a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return NULL; 4160a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell } 4170a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 4180a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* detach_buf clears data, so grab it now. */ 4190a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell ret = vq->data[i]; 4200a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell detach_buf(vq, i); 4210a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->last_used_idx++; 422a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin /* If we expect an interrupt for the next entry, tell host 423a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin * by writing event index and flush out the write before 424a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin * the read in the next get_buf call. */ 425a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) { 426a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin vring_used_event(&vq->vring) = vq->last_used_idx; 4277b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell virtio_mb(vq); 428a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin } 429a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin 4300a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell END_USE(vq); 4310a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return ret; 4320a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell} 4337c5e9ed0c84e7d70d887878574590638d5572659Michael S. TsirkinEXPORT_SYMBOL_GPL(virtqueue_get_buf); 4340a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 4355dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell/** 4365dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * virtqueue_disable_cb - disable callbacks 4375dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @vq: the struct virtqueue we're talking about. 4385dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 4395dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * Note that this is not necessarily synchronous, hence unreliable and only 4405dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * useful as an optimization. 4415dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 4425dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * Unlike other operations, this need not be serialized. 4435dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell */ 4447c5e9ed0c84e7d70d887878574590638d5572659Michael S. Tsirkinvoid virtqueue_disable_cb(struct virtqueue *_vq) 44518445c4d501b9ab4336f66ef46b092661ddaf336Rusty Russell{ 44618445c4d501b9ab4336f66ef46b092661ddaf336Rusty Russell struct vring_virtqueue *vq = to_vvq(_vq); 44718445c4d501b9ab4336f66ef46b092661ddaf336Rusty Russell 44818445c4d501b9ab4336f66ef46b092661ddaf336Rusty Russell vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 44918445c4d501b9ab4336f66ef46b092661ddaf336Rusty Russell} 4507c5e9ed0c84e7d70d887878574590638d5572659Michael S. TsirkinEXPORT_SYMBOL_GPL(virtqueue_disable_cb); 45118445c4d501b9ab4336f66ef46b092661ddaf336Rusty Russell 4525dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell/** 4535dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * virtqueue_enable_cb - restart callbacks after disable_cb. 4545dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @vq: the struct virtqueue we're talking about. 4555dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 4565dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * This re-enables callbacks; it returns "false" if there are pending 4575dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * buffers in the queue, to detect a possible race between the driver 4585dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * checking for more work, and enabling callbacks. 4595dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 4605dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * Caller must ensure we don't call this with other virtqueue 4615dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * operations at the same time (except where noted). 4625dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell */ 4637c5e9ed0c84e7d70d887878574590638d5572659Michael S. Tsirkinbool virtqueue_enable_cb(struct virtqueue *_vq) 4640a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell{ 4650a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell struct vring_virtqueue *vq = to_vvq(_vq); 4660a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 4670a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell START_USE(vq); 4680a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 4690a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* We optimistically turn back on interrupts, then check if there was 4700a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * more to do. */ 471a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to 472a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin * either clear the flags bit or point the event index at the next 473a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin * entry. Always do both to keep code simple. */ 4740a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 475a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin vring_used_event(&vq->vring) = vq->last_used_idx; 4767b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell virtio_mb(vq); 4770a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell if (unlikely(more_used(vq))) { 4780a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell END_USE(vq); 4790a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return false; 4800a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell } 4810a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 4820a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell END_USE(vq); 4830a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return true; 4840a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell} 4857c5e9ed0c84e7d70d887878574590638d5572659Michael S. TsirkinEXPORT_SYMBOL_GPL(virtqueue_enable_cb); 4860a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 4875dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell/** 4885dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * virtqueue_enable_cb_delayed - restart callbacks after disable_cb. 4895dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @vq: the struct virtqueue we're talking about. 4905dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 4915dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * This re-enables callbacks but hints to the other side to delay 4925dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * interrupts until most of the available buffers have been processed; 4935dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * it returns "false" if there are many pending buffers in the queue, 4945dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * to detect a possible race between the driver checking for more work, 4955dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * and enabling callbacks. 4965dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 4975dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * Caller must ensure we don't call this with other virtqueue 4985dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * operations at the same time (except where noted). 4995dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell */ 5007ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkinbool virtqueue_enable_cb_delayed(struct virtqueue *_vq) 5017ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin{ 5027ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin struct vring_virtqueue *vq = to_vvq(_vq); 5037ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin u16 bufs; 5047ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin 5057ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin START_USE(vq); 5067ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin 5077ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin /* We optimistically turn back on interrupts, then check if there was 5087ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin * more to do. */ 5097ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to 5107ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin * either clear the flags bit or point the event index at the next 5117ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin * entry. Always do both to keep code simple. */ 5127ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 5137ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin /* TODO: tune this threshold */ 5147ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4; 5157ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin vring_used_event(&vq->vring) = vq->last_used_idx + bufs; 5167b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell virtio_mb(vq); 5177ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) { 5187ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin END_USE(vq); 5197ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin return false; 5207ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin } 5217ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin 5227ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin END_USE(vq); 5237ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin return true; 5247ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin} 5257ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. TsirkinEXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed); 5267ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin 5275dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell/** 5285dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * virtqueue_detach_unused_buf - detach first unused buffer 5295dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @vq: the struct virtqueue we're talking about. 5305dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 531f96fde41f7f9af6cf20f6a1919f5d9670f84d574Rusty Russell * Returns NULL or the "data" token handed to virtqueue_add_buf(). 5325dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * This is not valid on an active queue; it is useful only for device 5335dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * shutdown. 5345dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell */ 5357c5e9ed0c84e7d70d887878574590638d5572659Michael S. Tsirkinvoid *virtqueue_detach_unused_buf(struct virtqueue *_vq) 536c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma{ 537c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma struct vring_virtqueue *vq = to_vvq(_vq); 538c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma unsigned int i; 539c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma void *buf; 540c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma 541c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma START_USE(vq); 542c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma 543c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma for (i = 0; i < vq->vring.num; i++) { 544c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma if (!vq->data[i]) 545c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma continue; 546c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma /* detach_buf clears data, so grab it now. */ 547c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma buf = vq->data[i]; 548c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma detach_buf(vq, i); 549b3258ff1d6086bd2b9eeb556844a868ad7d49bc8Amit Shah vq->vring.avail->idx--; 550c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma END_USE(vq); 551c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma return buf; 552c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma } 553c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma /* That should have freed everything. */ 554c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma BUG_ON(vq->num_free != vq->vring.num); 555c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma 556c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma END_USE(vq); 557c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma return NULL; 558c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma} 5597c5e9ed0c84e7d70d887878574590638d5572659Michael S. TsirkinEXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf); 560c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma 5610a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russellirqreturn_t vring_interrupt(int irq, void *_vq) 5620a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell{ 5630a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell struct vring_virtqueue *vq = to_vvq(_vq); 5640a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 5650a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell if (!more_used(vq)) { 5660a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell pr_debug("virtqueue interrupt with no work for %p\n", vq); 5670a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return IRQ_NONE; 5680a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell } 5690a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 5700a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell if (unlikely(vq->broken)) 5710a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return IRQ_HANDLED; 5720a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 5730a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback); 57418445c4d501b9ab4336f66ef46b092661ddaf336Rusty Russell if (vq->vq.callback) 57518445c4d501b9ab4336f66ef46b092661ddaf336Rusty Russell vq->vq.callback(&vq->vq); 5760a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 5770a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return IRQ_HANDLED; 5780a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell} 579c6fd47011b4bdebad3f1513bac75fe4895e332eeRusty RussellEXPORT_SYMBOL_GPL(vring_interrupt); 5800a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 5810a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russellstruct virtqueue *vring_new_virtqueue(unsigned int num, 58287c7d57c17ade5024d95b6ca0da249da49b0672aRusty Russell unsigned int vring_align, 5830a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell struct virtio_device *vdev, 5847b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell bool weak_barriers, 5850a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell void *pages, 5860a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell void (*notify)(struct virtqueue *), 5879499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell void (*callback)(struct virtqueue *), 5889499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell const char *name) 5890a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell{ 5900a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell struct vring_virtqueue *vq; 5910a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell unsigned int i; 5920a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 59342b36cc0ce717deeb10030141a43dede763a3ebeRusty Russell /* We assume num is a power of 2. */ 59442b36cc0ce717deeb10030141a43dede763a3ebeRusty Russell if (num & (num - 1)) { 59542b36cc0ce717deeb10030141a43dede763a3ebeRusty Russell dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num); 59642b36cc0ce717deeb10030141a43dede763a3ebeRusty Russell return NULL; 59742b36cc0ce717deeb10030141a43dede763a3ebeRusty Russell } 59842b36cc0ce717deeb10030141a43dede763a3ebeRusty Russell 5990a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq = kmalloc(sizeof(*vq) + sizeof(void *)*num, GFP_KERNEL); 6000a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell if (!vq) 6010a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return NULL; 6020a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 60387c7d57c17ade5024d95b6ca0da249da49b0672aRusty Russell vring_init(&vq->vring, num, pages, vring_align); 6040a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vq.callback = callback; 6050a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vq.vdev = vdev; 6069499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell vq->vq.name = name; 6070a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->notify = notify; 6087b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell vq->weak_barriers = weak_barriers; 6090a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->broken = false; 6100a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->last_used_idx = 0; 6110a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->num_added = 0; 6129499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell list_add_tail(&vq->vq.list, &vdev->vqs); 6130a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#ifdef DEBUG 6140a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->in_use = false; 6150a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#endif 6160a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 6179fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC); 618a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); 6199fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 6200a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* No callback? Tell other side not to bother us. */ 6210a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell if (!callback) 6220a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 6230a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 6240a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Put everything in free lists. */ 6250a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->num_free = num; 6260a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->free_head = 0; 6273b8706240ee6084ccb46e53cd3a554356b7eeec8Amit Shah for (i = 0; i < num-1; i++) { 6280a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vring.desc[i].next = i+1; 6293b8706240ee6084ccb46e53cd3a554356b7eeec8Amit Shah vq->data[i] = NULL; 6303b8706240ee6084ccb46e53cd3a554356b7eeec8Amit Shah } 6313b8706240ee6084ccb46e53cd3a554356b7eeec8Amit Shah vq->data[i] = NULL; 6320a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 6330a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return &vq->vq; 6340a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell} 635c6fd47011b4bdebad3f1513bac75fe4895e332eeRusty RussellEXPORT_SYMBOL_GPL(vring_new_virtqueue); 6360a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 6370a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russellvoid vring_del_virtqueue(struct virtqueue *vq) 6380a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell{ 6399499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell list_del(&vq->list); 6400a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell kfree(to_vvq(vq)); 6410a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell} 642c6fd47011b4bdebad3f1513bac75fe4895e332eeRusty RussellEXPORT_SYMBOL_GPL(vring_del_virtqueue); 6430a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 644e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell/* Manipulates transport-specific feature bits. */ 645e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russellvoid vring_transport_features(struct virtio_device *vdev) 646e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell{ 647e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell unsigned int i; 648e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell 649e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) { 650e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell switch (i) { 6519fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin case VIRTIO_RING_F_INDIRECT_DESC: 6529fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin break; 653a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin case VIRTIO_RING_F_EVENT_IDX: 654a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin break; 655e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell default: 656e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell /* We don't understand this bit. */ 657e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell clear_bit(i, vdev->features); 658e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell } 659e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell } 660e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell} 661e34f87256794b87e7f4a8f1812538be7b7b5214cRusty RussellEXPORT_SYMBOL_GPL(vring_transport_features); 662e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell 6635dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell/** 6645dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * virtqueue_get_vring_size - return the size of the virtqueue's vring 6655dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @vq: the struct virtqueue containing the vring of interest. 6665dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 6675dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * Returns the size of the vring. This is mainly used for boasting to 6685dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * userspace. Unlike other operations, this need not be serialized. 6695dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell */ 6708f9f4668b37bcc877156dd525a856055735c8d24Rick Jonesunsigned int virtqueue_get_vring_size(struct virtqueue *_vq) 6718f9f4668b37bcc877156dd525a856055735c8d24Rick Jones{ 6728f9f4668b37bcc877156dd525a856055735c8d24Rick Jones 6738f9f4668b37bcc877156dd525a856055735c8d24Rick Jones struct vring_virtqueue *vq = to_vvq(_vq); 6748f9f4668b37bcc877156dd525a856055735c8d24Rick Jones 6758f9f4668b37bcc877156dd525a856055735c8d24Rick Jones return vq->vring.num; 6768f9f4668b37bcc877156dd525a856055735c8d24Rick Jones} 6778f9f4668b37bcc877156dd525a856055735c8d24Rick JonesEXPORT_SYMBOL_GPL(virtqueue_get_vring_size); 6788f9f4668b37bcc877156dd525a856055735c8d24Rick Jones 679c6fd47011b4bdebad3f1513bac75fe4895e332eeRusty RussellMODULE_LICENSE("GPL"); 680