virtio_ring.c revision f96fde41f7f9af6cf20f6a1919f5d9670f84d574
10a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell/* Virtio ring implementation. 20a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * 30a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * Copyright 2007 Rusty Russell IBM Corporation 40a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * 50a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * This program is free software; you can redistribute it and/or modify 60a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * it under the terms of the GNU General Public License as published by 70a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * the Free Software Foundation; either version 2 of the License, or 80a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * (at your option) any later version. 90a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * 100a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * This program is distributed in the hope that it will be useful, 110a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * but WITHOUT ANY WARRANTY; without even the implied warranty of 120a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 130a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * GNU General Public License for more details. 140a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * 150a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * You should have received a copy of the GNU General Public License 160a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * along with this program; if not, write to the Free Software 170a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 180a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell */ 190a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#include <linux/virtio.h> 200a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#include <linux/virtio_ring.h> 21e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell#include <linux/virtio_config.h> 220a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#include <linux/device.h> 235a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 24b5a2c4f1996d1d258ac10897e0bbb57c6904a735Paul Gortmaker#include <linux/module.h> 250a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 26d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin/* virtio guest is communicating with a virtual "device" that actually runs on 27d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin * a host processor. Memory barriers are used to control SMP effects. */ 28d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin#ifdef CONFIG_SMP 29d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin/* Where possible, use SMP barriers which are more lightweight than mandatory 30d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin * barriers, because mandatory barriers control MMIO effects on accesses 317b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell * through relaxed memory I/O windows (which virtio-pci does not use). */ 327b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell#define virtio_mb(vq) \ 337b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell do { if ((vq)->weak_barriers) smp_mb(); else mb(); } while(0) 347b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell#define virtio_rmb(vq) \ 357b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0) 367b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell#define virtio_wmb(vq) \ 377b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0) 38d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin#else 39d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin/* We must force memory ordering even if guest is UP since host could be 40d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin * running on another CPU, but SMP barriers are defined to barrier() in that 41d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin * configuration. So fall back to mandatory barriers instead. */ 427b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell#define virtio_mb(vq) mb() 437b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell#define virtio_rmb(vq) rmb() 447b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell#define virtio_wmb(vq) wmb() 45d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin#endif 46d57ed95da483418e8b0433da693c9168dd0a2df6Michael S. Tsirkin 470a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#ifdef DEBUG 480a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell/* For development, we want to crash whenever the ring is screwed. */ 499499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell#define BAD_RING(_vq, fmt, args...) \ 509499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell do { \ 519499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell dev_err(&(_vq)->vq.vdev->dev, \ 529499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell "%s:"fmt, (_vq)->vq.name, ##args); \ 539499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell BUG(); \ 549499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell } while (0) 55c5f841f1780dad7efb7eca092f60742d47f47d25Rusty Russell/* Caller is supposed to guarantee no reentry. */ 56c5f841f1780dad7efb7eca092f60742d47f47d25Rusty Russell#define START_USE(_vq) \ 57c5f841f1780dad7efb7eca092f60742d47f47d25Rusty Russell do { \ 58c5f841f1780dad7efb7eca092f60742d47f47d25Rusty Russell if ((_vq)->in_use) \ 599499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell panic("%s:in_use = %i\n", \ 609499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell (_vq)->vq.name, (_vq)->in_use); \ 61c5f841f1780dad7efb7eca092f60742d47f47d25Rusty Russell (_vq)->in_use = __LINE__; \ 629499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell } while (0) 633a35ce7dcefe9e80a00603a195269fbaf6e7d901Roel Kluin#define END_USE(_vq) \ 6497a545ab6ce922a0f868d192718a48a0091ebc5eRusty Russell do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0) 650a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#else 669499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell#define BAD_RING(_vq, fmt, args...) \ 679499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell do { \ 689499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell dev_err(&_vq->vq.vdev->dev, \ 699499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell "%s:"fmt, (_vq)->vq.name, ##args); \ 709499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell (_vq)->broken = true; \ 719499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell } while (0) 720a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#define START_USE(vq) 730a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#define END_USE(vq) 740a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#endif 750a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 760a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russellstruct vring_virtqueue 770a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell{ 780a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell struct virtqueue vq; 790a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 800a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Actual memory layout for this queue */ 810a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell struct vring vring; 820a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 837b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell /* Can we use weak barriers? */ 847b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell bool weak_barriers; 857b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell 860a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Other side has made a mess, don't try any more. */ 870a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell bool broken; 880a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 899fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin /* Host supports indirect buffers */ 909fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin bool indirect; 919fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 92a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin /* Host publishes avail event idx */ 93a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin bool event; 94a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin 950a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Number of free buffers */ 960a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell unsigned int num_free; 970a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Head of free buffer list. */ 980a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell unsigned int free_head; 990a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Number we've added since last sync. */ 1000a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell unsigned int num_added; 1010a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 1020a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Last used index we've seen. */ 1031bc4953ed44454c7f53d0b609445d1534981ee75Anthony Liguori u16 last_used_idx; 1040a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 1050a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* How to notify other side. FIXME: commonalize hcalls! */ 1060a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell void (*notify)(struct virtqueue *vq); 1070a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 1080a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#ifdef DEBUG 1090a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* They're supposed to lock for us. */ 1100a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell unsigned int in_use; 1110a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#endif 1120a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 1130a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Tokens for callbacks. */ 1140a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell void *data[]; 1150a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell}; 1160a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 1170a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) 1180a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 1199fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin/* Set up an indirect table of descriptors and add it to the queue. */ 1209fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlinstatic int vring_add_indirect(struct vring_virtqueue *vq, 1219fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin struct scatterlist sg[], 1229fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin unsigned int out, 123bbd603efb4238cf78083c00f0a81adfa8994aa33Michael S. Tsirkin unsigned int in, 124bbd603efb4238cf78083c00f0a81adfa8994aa33Michael S. Tsirkin gfp_t gfp) 1259fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin{ 1269fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin struct vring_desc *desc; 1279fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin unsigned head; 1289fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin int i; 1299fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 130bbd603efb4238cf78083c00f0a81adfa8994aa33Michael S. Tsirkin desc = kmalloc((out + in) * sizeof(struct vring_desc), gfp); 1319fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin if (!desc) 132686d363786a53ed28ee875b84ef24e6d5126ef6fMichael S. Tsirkin return -ENOMEM; 1339fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 1349fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin /* Transfer entries from the sg list into the indirect page */ 1359fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin for (i = 0; i < out; i++) { 1369fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin desc[i].flags = VRING_DESC_F_NEXT; 1379fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin desc[i].addr = sg_phys(sg); 1389fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin desc[i].len = sg->length; 1399fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin desc[i].next = i+1; 1409fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin sg++; 1419fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin } 1429fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin for (; i < (out + in); i++) { 1439fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin desc[i].flags = VRING_DESC_F_NEXT|VRING_DESC_F_WRITE; 1449fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin desc[i].addr = sg_phys(sg); 1459fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin desc[i].len = sg->length; 1469fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin desc[i].next = i+1; 1479fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin sg++; 1489fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin } 1499fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 1509fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin /* Last one doesn't continue. */ 1519fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin desc[i-1].flags &= ~VRING_DESC_F_NEXT; 1529fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin desc[i-1].next = 0; 1539fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 1549fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin /* We're about to use a buffer */ 1559fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin vq->num_free--; 1569fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 1579fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin /* Use a single buffer which doesn't continue */ 1589fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin head = vq->free_head; 1599fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin vq->vring.desc[head].flags = VRING_DESC_F_INDIRECT; 1609fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin vq->vring.desc[head].addr = virt_to_phys(desc); 1619fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin vq->vring.desc[head].len = i * sizeof(struct vring_desc); 1629fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 1639fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin /* Update free pointer */ 1649fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin vq->free_head = vq->vring.desc[head].next; 1659fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 1669fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin return head; 1679fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin} 1689fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 1695dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell/** 170f96fde41f7f9af6cf20f6a1919f5d9670f84d574Rusty Russell * virtqueue_add_buf - expose buffer to other end 1715dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @vq: the struct virtqueue we're talking about. 1725dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @sg: the description of the buffer(s). 1735dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @out_num: the number of sg readable by other side 1745dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @in_num: the number of sg which are writable (after readable ones) 1755dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @data: the token identifying the buffer. 1765dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @gfp: how to do memory allocations (if necessary). 1775dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 1785dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * Caller must ensure we don't call this with other virtqueue operations 1795dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * at the same time (except where noted). 1805dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 1815dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * Returns remaining capacity of queue or a negative error 1825dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * (ie. ENOSPC). Note that it only really makes sense to treat all 1835dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * positive return values as "available": indirect buffers mean that 1845dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * we can put an entire sg[] array inside a single queue entry. 1855dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell */ 186f96fde41f7f9af6cf20f6a1919f5d9670f84d574Rusty Russellint virtqueue_add_buf(struct virtqueue *_vq, 187f96fde41f7f9af6cf20f6a1919f5d9670f84d574Rusty Russell struct scatterlist sg[], 188f96fde41f7f9af6cf20f6a1919f5d9670f84d574Rusty Russell unsigned int out, 189f96fde41f7f9af6cf20f6a1919f5d9670f84d574Rusty Russell unsigned int in, 190f96fde41f7f9af6cf20f6a1919f5d9670f84d574Rusty Russell void *data, 191f96fde41f7f9af6cf20f6a1919f5d9670f84d574Rusty Russell gfp_t gfp) 1920a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell{ 1930a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell struct vring_virtqueue *vq = to_vvq(_vq); 1941fe9b6fef11771461e69ecd1bc8935a1c7c90cb5Michael S. Tsirkin unsigned int i, avail, uninitialized_var(prev); 1951fe9b6fef11771461e69ecd1bc8935a1c7c90cb5Michael S. Tsirkin int head; 1960a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 1979fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin START_USE(vq); 1989fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 1990a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell BUG_ON(data == NULL); 2009fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 2019fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin /* If the host supports indirect descriptor tables, and we have multiple 2029fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin * buffers, then go indirect. FIXME: tune this threshold */ 2039fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin if (vq->indirect && (out + in) > 1 && vq->num_free) { 204bbd603efb4238cf78083c00f0a81adfa8994aa33Michael S. Tsirkin head = vring_add_indirect(vq, sg, out, in, gfp); 2051fe9b6fef11771461e69ecd1bc8935a1c7c90cb5Michael S. Tsirkin if (likely(head >= 0)) 2069fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin goto add_head; 2079fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin } 2089fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 2090a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell BUG_ON(out + in > vq->vring.num); 2100a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell BUG_ON(out + in == 0); 2110a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 2120a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell if (vq->num_free < out + in) { 2130a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell pr_debug("Can't add buf len %i - avail = %i\n", 2140a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell out + in, vq->num_free); 21544653eae1407f79dff6f52fcf594ae84cb165ec4Rusty Russell /* FIXME: for historical reasons, we force a notify here if 21644653eae1407f79dff6f52fcf594ae84cb165ec4Rusty Russell * there are outgoing parts to the buffer. Presumably the 21744653eae1407f79dff6f52fcf594ae84cb165ec4Rusty Russell * host should service the ring ASAP. */ 21844653eae1407f79dff6f52fcf594ae84cb165ec4Rusty Russell if (out) 21944653eae1407f79dff6f52fcf594ae84cb165ec4Rusty Russell vq->notify(&vq->vq); 2200a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell END_USE(vq); 2210a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return -ENOSPC; 2220a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell } 2230a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 2240a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* We're about to use some buffers from the free list. */ 2250a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->num_free -= out + in; 2260a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 2270a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell head = vq->free_head; 2280a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell for (i = vq->free_head; out; i = vq->vring.desc[i].next, out--) { 2290a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vring.desc[i].flags = VRING_DESC_F_NEXT; 23015f9c8903cbdb02aee0f1bcf86a97c2e238b9a3dRusty Russell vq->vring.desc[i].addr = sg_phys(sg); 2310a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vring.desc[i].len = sg->length; 2320a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell prev = i; 2330a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell sg++; 2340a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell } 2350a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell for (; in; i = vq->vring.desc[i].next, in--) { 2360a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vring.desc[i].flags = VRING_DESC_F_NEXT|VRING_DESC_F_WRITE; 23715f9c8903cbdb02aee0f1bcf86a97c2e238b9a3dRusty Russell vq->vring.desc[i].addr = sg_phys(sg); 2380a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vring.desc[i].len = sg->length; 2390a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell prev = i; 2400a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell sg++; 2410a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell } 2420a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Last one doesn't continue. */ 2430a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vring.desc[prev].flags &= ~VRING_DESC_F_NEXT; 2440a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 2450a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Update free pointer */ 2460a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->free_head = i; 2470a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 2489fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlinadd_head: 2490a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Set token. */ 2500a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->data[head] = data; 2510a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 2520a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Put entry in available array (but don't update avail->idx until they 2530a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * do sync). FIXME: avoid modulus here? */ 2540a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell avail = (vq->vring.avail->idx + vq->num_added++) % vq->vring.num; 2550a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vring.avail->ring[avail] = head; 2560a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 2570a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell pr_debug("Added buffer head %i to %p\n", head, vq); 2580a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell END_USE(vq); 2593c1b27d5043086a485f8526353ae9fe37bfa1065Rusty Russell 2603c1b27d5043086a485f8526353ae9fe37bfa1065Rusty Russell return vq->num_free; 2610a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell} 262f96fde41f7f9af6cf20f6a1919f5d9670f84d574Rusty RussellEXPORT_SYMBOL_GPL(virtqueue_add_buf); 2630a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 2645dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell/** 2655dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * virtqueue_kick - update after add_buf 2665dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @vq: the struct virtqueue 2675dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 268f96fde41f7f9af6cf20f6a1919f5d9670f84d574Rusty Russell * After one or more virtqueue_add_buf calls, invoke this to kick 2695dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * the other side. 2705dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 2715dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * Caller must ensure we don't call this with other virtqueue 2725dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * operations at the same time (except where noted). 2735dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell */ 2747c5e9ed0c84e7d70d887878574590638d5572659Michael S. Tsirkinvoid virtqueue_kick(struct virtqueue *_vq) 2750a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell{ 2760a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell struct vring_virtqueue *vq = to_vvq(_vq); 277a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin u16 new, old; 2780a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell START_USE(vq); 2790a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Descriptors and available array need to be set before we expose the 2800a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * new available array entries. */ 2817b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell virtio_wmb(vq); 2820a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 283a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin old = vq->vring.avail->idx; 284a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin new = vq->vring.avail->idx = old + vq->num_added; 2850a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->num_added = 0; 2860a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 2870a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Need to update avail index before checking if we should notify */ 2887b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell virtio_mb(vq); 2890a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 290a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin if (vq->event ? 291a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin vring_need_event(vring_avail_event(&vq->vring), new, old) : 292a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin !(vq->vring.used->flags & VRING_USED_F_NO_NOTIFY)) 2930a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Prod other side to tell it about changes. */ 2940a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->notify(&vq->vq); 2950a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 2960a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell END_USE(vq); 2970a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell} 2987c5e9ed0c84e7d70d887878574590638d5572659Michael S. TsirkinEXPORT_SYMBOL_GPL(virtqueue_kick); 2990a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 3000a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russellstatic void detach_buf(struct vring_virtqueue *vq, unsigned int head) 3010a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell{ 3020a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell unsigned int i; 3030a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 3040a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Clear data ptr. */ 3050a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->data[head] = NULL; 3060a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 3070a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Put back on free list: find end */ 3080a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell i = head; 3099fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 3109fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin /* Free the indirect table */ 3119fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin if (vq->vring.desc[i].flags & VRING_DESC_F_INDIRECT) 3129fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin kfree(phys_to_virt(vq->vring.desc[i].addr)); 3139fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 3140a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell while (vq->vring.desc[i].flags & VRING_DESC_F_NEXT) { 3150a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell i = vq->vring.desc[i].next; 3160a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->num_free++; 3170a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell } 3180a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 3190a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vring.desc[i].next = vq->free_head; 3200a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->free_head = head; 3210a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Plus final descriptor */ 3220a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->num_free++; 3230a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell} 3240a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 3250a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russellstatic inline bool more_used(const struct vring_virtqueue *vq) 3260a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell{ 3270a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return vq->last_used_idx != vq->vring.used->idx; 3280a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell} 3290a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 3305dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell/** 3315dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * virtqueue_get_buf - get the next used buffer 3325dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @vq: the struct virtqueue we're talking about. 3335dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @len: the length written into the buffer 3345dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 3355dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * If the driver wrote data into the buffer, @len will be set to the 3365dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * amount written. This means you don't need to clear the buffer 3375dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * beforehand to ensure there's no data leakage in the case of short 3385dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * writes. 3395dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 3405dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * Caller must ensure we don't call this with other virtqueue 3415dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * operations at the same time (except where noted). 3425dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 3435dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * Returns NULL if there are no used buffers, or the "data" token 344f96fde41f7f9af6cf20f6a1919f5d9670f84d574Rusty Russell * handed to virtqueue_add_buf(). 3455dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell */ 3467c5e9ed0c84e7d70d887878574590638d5572659Michael S. Tsirkinvoid *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len) 3470a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell{ 3480a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell struct vring_virtqueue *vq = to_vvq(_vq); 3490a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell void *ret; 3500a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell unsigned int i; 3510a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 3520a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell START_USE(vq); 3530a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 3545ef827526fc01820a7a80827802e9fad3f34f937Rusty Russell if (unlikely(vq->broken)) { 3555ef827526fc01820a7a80827802e9fad3f34f937Rusty Russell END_USE(vq); 3565ef827526fc01820a7a80827802e9fad3f34f937Rusty Russell return NULL; 3575ef827526fc01820a7a80827802e9fad3f34f937Rusty Russell } 3585ef827526fc01820a7a80827802e9fad3f34f937Rusty Russell 3590a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell if (!more_used(vq)) { 3600a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell pr_debug("No more buffers in queue\n"); 3610a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell END_USE(vq); 3620a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return NULL; 3630a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell } 3640a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 3652d61ba95034f1abbdec7729d52c740870a5eddb6Michael S. Tsirkin /* Only get used array entries after they have been exposed by host. */ 3667b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell virtio_rmb(vq); 3672d61ba95034f1abbdec7729d52c740870a5eddb6Michael S. Tsirkin 3680a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id; 3690a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell *len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len; 3700a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 3710a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell if (unlikely(i >= vq->vring.num)) { 3720a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell BAD_RING(vq, "id %u out of range\n", i); 3730a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return NULL; 3740a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell } 3750a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell if (unlikely(!vq->data[i])) { 3760a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell BAD_RING(vq, "id %u is not a head!\n", i); 3770a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return NULL; 3780a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell } 3790a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 3800a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* detach_buf clears data, so grab it now. */ 3810a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell ret = vq->data[i]; 3820a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell detach_buf(vq, i); 3830a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->last_used_idx++; 384a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin /* If we expect an interrupt for the next entry, tell host 385a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin * by writing event index and flush out the write before 386a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin * the read in the next get_buf call. */ 387a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) { 388a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin vring_used_event(&vq->vring) = vq->last_used_idx; 3897b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell virtio_mb(vq); 390a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin } 391a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin 3920a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell END_USE(vq); 3930a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return ret; 3940a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell} 3957c5e9ed0c84e7d70d887878574590638d5572659Michael S. TsirkinEXPORT_SYMBOL_GPL(virtqueue_get_buf); 3960a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 3975dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell/** 3985dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * virtqueue_disable_cb - disable callbacks 3995dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @vq: the struct virtqueue we're talking about. 4005dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 4015dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * Note that this is not necessarily synchronous, hence unreliable and only 4025dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * useful as an optimization. 4035dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 4045dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * Unlike other operations, this need not be serialized. 4055dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell */ 4067c5e9ed0c84e7d70d887878574590638d5572659Michael S. Tsirkinvoid virtqueue_disable_cb(struct virtqueue *_vq) 40718445c4d501b9ab4336f66ef46b092661ddaf336Rusty Russell{ 40818445c4d501b9ab4336f66ef46b092661ddaf336Rusty Russell struct vring_virtqueue *vq = to_vvq(_vq); 40918445c4d501b9ab4336f66ef46b092661ddaf336Rusty Russell 41018445c4d501b9ab4336f66ef46b092661ddaf336Rusty Russell vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 41118445c4d501b9ab4336f66ef46b092661ddaf336Rusty Russell} 4127c5e9ed0c84e7d70d887878574590638d5572659Michael S. TsirkinEXPORT_SYMBOL_GPL(virtqueue_disable_cb); 41318445c4d501b9ab4336f66ef46b092661ddaf336Rusty Russell 4145dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell/** 4155dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * virtqueue_enable_cb - restart callbacks after disable_cb. 4165dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @vq: the struct virtqueue we're talking about. 4175dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 4185dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * This re-enables callbacks; it returns "false" if there are pending 4195dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * buffers in the queue, to detect a possible race between the driver 4205dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * checking for more work, and enabling callbacks. 4215dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 4225dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * Caller must ensure we don't call this with other virtqueue 4235dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * operations at the same time (except where noted). 4245dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell */ 4257c5e9ed0c84e7d70d887878574590638d5572659Michael S. Tsirkinbool virtqueue_enable_cb(struct virtqueue *_vq) 4260a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell{ 4270a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell struct vring_virtqueue *vq = to_vvq(_vq); 4280a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 4290a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell START_USE(vq); 4300a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 4310a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* We optimistically turn back on interrupts, then check if there was 4320a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell * more to do. */ 433a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to 434a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin * either clear the flags bit or point the event index at the next 435a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin * entry. Always do both to keep code simple. */ 4360a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 437a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin vring_used_event(&vq->vring) = vq->last_used_idx; 4387b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell virtio_mb(vq); 4390a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell if (unlikely(more_used(vq))) { 4400a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell END_USE(vq); 4410a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return false; 4420a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell } 4430a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 4440a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell END_USE(vq); 4450a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return true; 4460a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell} 4477c5e9ed0c84e7d70d887878574590638d5572659Michael S. TsirkinEXPORT_SYMBOL_GPL(virtqueue_enable_cb); 4480a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 4495dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell/** 4505dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * virtqueue_enable_cb_delayed - restart callbacks after disable_cb. 4515dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @vq: the struct virtqueue we're talking about. 4525dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 4535dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * This re-enables callbacks but hints to the other side to delay 4545dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * interrupts until most of the available buffers have been processed; 4555dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * it returns "false" if there are many pending buffers in the queue, 4565dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * to detect a possible race between the driver checking for more work, 4575dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * and enabling callbacks. 4585dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 4595dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * Caller must ensure we don't call this with other virtqueue 4605dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * operations at the same time (except where noted). 4615dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell */ 4627ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkinbool virtqueue_enable_cb_delayed(struct virtqueue *_vq) 4637ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin{ 4647ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin struct vring_virtqueue *vq = to_vvq(_vq); 4657ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin u16 bufs; 4667ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin 4677ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin START_USE(vq); 4687ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin 4697ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin /* We optimistically turn back on interrupts, then check if there was 4707ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin * more to do. */ 4717ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to 4727ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin * either clear the flags bit or point the event index at the next 4737ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin * entry. Always do both to keep code simple. */ 4747ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 4757ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin /* TODO: tune this threshold */ 4767ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4; 4777ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin vring_used_event(&vq->vring) = vq->last_used_idx + bufs; 4787b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell virtio_mb(vq); 4797ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) { 4807ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin END_USE(vq); 4817ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin return false; 4827ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin } 4837ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin 4847ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin END_USE(vq); 4857ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin return true; 4867ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin} 4877ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. TsirkinEXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed); 4887ab358c23cbf15cea08129cd722d1ce77433a94dMichael S. Tsirkin 4895dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell/** 4905dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * virtqueue_detach_unused_buf - detach first unused buffer 4915dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @vq: the struct virtqueue we're talking about. 4925dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 493f96fde41f7f9af6cf20f6a1919f5d9670f84d574Rusty Russell * Returns NULL or the "data" token handed to virtqueue_add_buf(). 4945dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * This is not valid on an active queue; it is useful only for device 4955dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * shutdown. 4965dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell */ 4977c5e9ed0c84e7d70d887878574590638d5572659Michael S. Tsirkinvoid *virtqueue_detach_unused_buf(struct virtqueue *_vq) 498c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma{ 499c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma struct vring_virtqueue *vq = to_vvq(_vq); 500c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma unsigned int i; 501c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma void *buf; 502c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma 503c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma START_USE(vq); 504c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma 505c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma for (i = 0; i < vq->vring.num; i++) { 506c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma if (!vq->data[i]) 507c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma continue; 508c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma /* detach_buf clears data, so grab it now. */ 509c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma buf = vq->data[i]; 510c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma detach_buf(vq, i); 511b3258ff1d6086bd2b9eeb556844a868ad7d49bc8Amit Shah vq->vring.avail->idx--; 512c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma END_USE(vq); 513c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma return buf; 514c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma } 515c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma /* That should have freed everything. */ 516c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma BUG_ON(vq->num_free != vq->vring.num); 517c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma 518c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma END_USE(vq); 519c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma return NULL; 520c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma} 5217c5e9ed0c84e7d70d887878574590638d5572659Michael S. TsirkinEXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf); 522c021eac4148c16bf53baa0dd14e8ebee6f39dab5Shirley Ma 5230a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russellirqreturn_t vring_interrupt(int irq, void *_vq) 5240a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell{ 5250a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell struct vring_virtqueue *vq = to_vvq(_vq); 5260a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 5270a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell if (!more_used(vq)) { 5280a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell pr_debug("virtqueue interrupt with no work for %p\n", vq); 5290a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return IRQ_NONE; 5300a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell } 5310a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 5320a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell if (unlikely(vq->broken)) 5330a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return IRQ_HANDLED; 5340a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 5350a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback); 53618445c4d501b9ab4336f66ef46b092661ddaf336Rusty Russell if (vq->vq.callback) 53718445c4d501b9ab4336f66ef46b092661ddaf336Rusty Russell vq->vq.callback(&vq->vq); 5380a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 5390a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return IRQ_HANDLED; 5400a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell} 541c6fd47011b4bdebad3f1513bac75fe4895e332eeRusty RussellEXPORT_SYMBOL_GPL(vring_interrupt); 5420a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 5430a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russellstruct virtqueue *vring_new_virtqueue(unsigned int num, 54487c7d57c17ade5024d95b6ca0da249da49b0672aRusty Russell unsigned int vring_align, 5450a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell struct virtio_device *vdev, 5467b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell bool weak_barriers, 5470a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell void *pages, 5480a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell void (*notify)(struct virtqueue *), 5499499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell void (*callback)(struct virtqueue *), 5509499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell const char *name) 5510a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell{ 5520a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell struct vring_virtqueue *vq; 5530a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell unsigned int i; 5540a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 55542b36cc0ce717deeb10030141a43dede763a3ebeRusty Russell /* We assume num is a power of 2. */ 55642b36cc0ce717deeb10030141a43dede763a3ebeRusty Russell if (num & (num - 1)) { 55742b36cc0ce717deeb10030141a43dede763a3ebeRusty Russell dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num); 55842b36cc0ce717deeb10030141a43dede763a3ebeRusty Russell return NULL; 55942b36cc0ce717deeb10030141a43dede763a3ebeRusty Russell } 56042b36cc0ce717deeb10030141a43dede763a3ebeRusty Russell 5610a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq = kmalloc(sizeof(*vq) + sizeof(void *)*num, GFP_KERNEL); 5620a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell if (!vq) 5630a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return NULL; 5640a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 56587c7d57c17ade5024d95b6ca0da249da49b0672aRusty Russell vring_init(&vq->vring, num, pages, vring_align); 5660a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vq.callback = callback; 5670a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vq.vdev = vdev; 5689499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell vq->vq.name = name; 5690a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->notify = notify; 5707b21e34fd1c272e3a8c3846168f2f6287a4cd72bRusty Russell vq->weak_barriers = weak_barriers; 5710a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->broken = false; 5720a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->last_used_idx = 0; 5730a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->num_added = 0; 5749499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell list_add_tail(&vq->vq.list, &vdev->vqs); 5750a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#ifdef DEBUG 5760a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->in_use = false; 5770a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell#endif 5780a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 5799fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC); 580a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); 5819fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin 5820a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* No callback? Tell other side not to bother us. */ 5830a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell if (!callback) 5840a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 5850a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 5860a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell /* Put everything in free lists. */ 5870a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->num_free = num; 5880a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->free_head = 0; 5893b8706240ee6084ccb46e53cd3a554356b7eeec8Amit Shah for (i = 0; i < num-1; i++) { 5900a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell vq->vring.desc[i].next = i+1; 5913b8706240ee6084ccb46e53cd3a554356b7eeec8Amit Shah vq->data[i] = NULL; 5923b8706240ee6084ccb46e53cd3a554356b7eeec8Amit Shah } 5933b8706240ee6084ccb46e53cd3a554356b7eeec8Amit Shah vq->data[i] = NULL; 5940a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 5950a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell return &vq->vq; 5960a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell} 597c6fd47011b4bdebad3f1513bac75fe4895e332eeRusty RussellEXPORT_SYMBOL_GPL(vring_new_virtqueue); 5980a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 5990a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russellvoid vring_del_virtqueue(struct virtqueue *vq) 6000a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell{ 6019499f5e7ed5224c40706f0cec6542a9916bc7606Rusty Russell list_del(&vq->list); 6020a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell kfree(to_vvq(vq)); 6030a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell} 604c6fd47011b4bdebad3f1513bac75fe4895e332eeRusty RussellEXPORT_SYMBOL_GPL(vring_del_virtqueue); 6050a8a69dd77ddbd4513b21363021ecde7e1025502Rusty Russell 606e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell/* Manipulates transport-specific feature bits. */ 607e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russellvoid vring_transport_features(struct virtio_device *vdev) 608e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell{ 609e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell unsigned int i; 610e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell 611e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) { 612e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell switch (i) { 6139fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin case VIRTIO_RING_F_INDIRECT_DESC: 6149fa29b9df32ba4db055f3977933cd0c1b8fe67cdMark McLoughlin break; 615a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin case VIRTIO_RING_F_EVENT_IDX: 616a5c262c5fd83ece01bd649fb08416c501d4c59d7Michael S. Tsirkin break; 617e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell default: 618e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell /* We don't understand this bit. */ 619e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell clear_bit(i, vdev->features); 620e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell } 621e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell } 622e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell} 623e34f87256794b87e7f4a8f1812538be7b7b5214cRusty RussellEXPORT_SYMBOL_GPL(vring_transport_features); 624e34f87256794b87e7f4a8f1812538be7b7b5214cRusty Russell 6255dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell/** 6265dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * virtqueue_get_vring_size - return the size of the virtqueue's vring 6275dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * @vq: the struct virtqueue containing the vring of interest. 6285dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * 6295dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * Returns the size of the vring. This is mainly used for boasting to 6305dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell * userspace. Unlike other operations, this need not be serialized. 6315dfc17628d57f9e62043ed0cba03a6e3eb019a78Rusty Russell */ 6328f9f4668b37bcc877156dd525a856055735c8d24Rick Jonesunsigned int virtqueue_get_vring_size(struct virtqueue *_vq) 6338f9f4668b37bcc877156dd525a856055735c8d24Rick Jones{ 6348f9f4668b37bcc877156dd525a856055735c8d24Rick Jones 6358f9f4668b37bcc877156dd525a856055735c8d24Rick Jones struct vring_virtqueue *vq = to_vvq(_vq); 6368f9f4668b37bcc877156dd525a856055735c8d24Rick Jones 6378f9f4668b37bcc877156dd525a856055735c8d24Rick Jones return vq->vring.num; 6388f9f4668b37bcc877156dd525a856055735c8d24Rick Jones} 6398f9f4668b37bcc877156dd525a856055735c8d24Rick JonesEXPORT_SYMBOL_GPL(virtqueue_get_vring_size); 6408f9f4668b37bcc877156dd525a856055735c8d24Rick Jones 641c6fd47011b4bdebad3f1513bac75fe4895e332eeRusty RussellMODULE_LICENSE("GPL"); 642