1cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan/* 2e7eacd36865ae0707f5efae8e4dda421ffcd1b66Ralph Campbell * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved. 3cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. 4cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * 5cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * This software is available to you under a choice of one of two 6cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * licenses. You may choose to be licensed under the terms of the GNU 7cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * General Public License (GPL) Version 2, available from the file 8cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * COPYING in the main directory of this source tree, or the 9cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * OpenIB.org BSD license below: 10cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * 11cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * Redistribution and use in source and binary forms, with or 12cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * without modification, are permitted provided that the following 13cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * conditions are met: 14cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * 15cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * - Redistributions of source code must retain the above 16cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * copyright notice, this list of conditions and the following 17cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * disclaimer. 18cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * 19cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * - Redistributions in binary form must reproduce the above 20cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * copyright notice, this list of conditions and the following 21cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * disclaimer in the documentation and/or other materials 22cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * provided with the distribution. 23cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * 24cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * SOFTWARE. 32cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan */ 33cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 34cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan#include <linux/err.h> 355a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 36cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan#include <linux/vmalloc.h> 37cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 38cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan#include "ipath_verbs.h" 39cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 40cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan/** 41cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * ipath_post_srq_receive - post a receive on a shared receive queue 42cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * @ibsrq: the SRQ to post the receive on 43cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * @wr: the list of work requests to post 44cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * @bad_wr: the first WR to cause a problem is put here 45cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * 46cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * This may be called from interrupt context. 47cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan */ 48cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivanint ipath_post_srq_receive(struct ib_srq *ibsrq, struct ib_recv_wr *wr, 49cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan struct ib_recv_wr **bad_wr) 50cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan{ 51cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan struct ipath_srq *srq = to_isrq(ibsrq); 52373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell struct ipath_rwq *wq; 53cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan unsigned long flags; 54cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan int ret; 55cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 56cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan for (; wr; wr = wr->next) { 57cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan struct ipath_rwqe *wqe; 58cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan u32 next; 59373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell int i; 60cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 61373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if ((unsigned) wr->num_sge > srq->rq.max_sge) { 62cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan *bad_wr = wr; 634187b915a0f7eaa69707715e80d9fc253ff6167aRalph Campbell ret = -EINVAL; 64cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan goto bail; 65cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan } 66cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 67cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan spin_lock_irqsave(&srq->rq.lock, flags); 68373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell wq = srq->rq.wq; 69373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell next = wq->head + 1; 70cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan if (next >= srq->rq.size) 71cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan next = 0; 72373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if (next == wq->tail) { 73cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan spin_unlock_irqrestore(&srq->rq.lock, flags); 74cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan *bad_wr = wr; 75cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan ret = -ENOMEM; 76cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan goto bail; 77cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan } 78cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 79373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell wqe = get_rwqe_ptr(&srq->rq, wq->head); 80cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan wqe->wr_id = wr->wr_id; 81373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell wqe->num_sge = wr->num_sge; 82373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell for (i = 0; i < wr->num_sge; i++) 83373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell wqe->sg_list[i] = wr->sg_list[i]; 844fc570bcbe77f823aae183dd824869f79e74cc97Ralph Campbell /* Make sure queue entry is written before the head index. */ 854fc570bcbe77f823aae183dd824869f79e74cc97Ralph Campbell smp_wmb(); 86373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell wq->head = next; 87cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan spin_unlock_irqrestore(&srq->rq.lock, flags); 88cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan } 89cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan ret = 0; 90cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 91cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivanbail: 92cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan return ret; 93cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan} 94cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 95cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan/** 96cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * ipath_create_srq - create a shared receive queue 97cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * @ibpd: the protection domain of the SRQ to create 986276980138292056aab162f7eb98afe5243b2976Ralph Campbell * @srq_init_attr: the attributes of the SRQ 996276980138292056aab162f7eb98afe5243b2976Ralph Campbell * @udata: data from libipathverbs when creating a user SRQ 100cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan */ 101cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivanstruct ib_srq *ipath_create_srq(struct ib_pd *ibpd, 102cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan struct ib_srq_init_attr *srq_init_attr, 103cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan struct ib_udata *udata) 104cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan{ 105fe62546a6afa141c4ab9aef65f5978a1b36cb523Bryan O'Sullivan struct ipath_ibdev *dev = to_idev(ibpd->device); 106cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan struct ipath_srq *srq; 107cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan u32 sz; 108cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan struct ib_srq *ret; 109cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 11096104eda01695a26da2c8f7423ec0ba3509c8c97Sean Hefty if (srq_init_attr->srq_type != IB_SRQT_BASIC) { 11196104eda01695a26da2c8f7423ec0ba3509c8c97Sean Hefty ret = ERR_PTR(-ENOSYS); 11296104eda01695a26da2c8f7423ec0ba3509c8c97Sean Hefty goto done; 11396104eda01695a26da2c8f7423ec0ba3509c8c97Sean Hefty } 11496104eda01695a26da2c8f7423ec0ba3509c8c97Sean Hefty 115fe62546a6afa141c4ab9aef65f5978a1b36cb523Bryan O'Sullivan if (srq_init_attr->attr.max_wr == 0) { 116fe62546a6afa141c4ab9aef65f5978a1b36cb523Bryan O'Sullivan ret = ERR_PTR(-EINVAL); 117373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell goto done; 118fe62546a6afa141c4ab9aef65f5978a1b36cb523Bryan O'Sullivan } 119fe62546a6afa141c4ab9aef65f5978a1b36cb523Bryan O'Sullivan 120fe62546a6afa141c4ab9aef65f5978a1b36cb523Bryan O'Sullivan if ((srq_init_attr->attr.max_sge > ib_ipath_max_srq_sges) || 121fe62546a6afa141c4ab9aef65f5978a1b36cb523Bryan O'Sullivan (srq_init_attr->attr.max_wr > ib_ipath_max_srq_wrs)) { 122cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan ret = ERR_PTR(-EINVAL); 123373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell goto done; 124cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan } 125cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 126cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan srq = kmalloc(sizeof(*srq), GFP_KERNEL); 127cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan if (!srq) { 128cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan ret = ERR_PTR(-ENOMEM); 129373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell goto done; 130cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan } 131cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 132cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan /* 133cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * Need to use vmalloc() if we want to support large #s of entries. 134cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan */ 135cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan srq->rq.size = srq_init_attr->attr.max_wr + 1; 136373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell srq->rq.max_sge = srq_init_attr->attr.max_sge; 137373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell sz = sizeof(struct ib_sge) * srq->rq.max_sge + 138cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan sizeof(struct ipath_rwqe); 139373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell srq->rq.wq = vmalloc_user(sizeof(struct ipath_rwq) + srq->rq.size * sz); 140cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan if (!srq->rq.wq) { 141cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan ret = ERR_PTR(-ENOMEM); 142373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell goto bail_srq; 143cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan } 144cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 145cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan /* 146373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell * Return the address of the RWQ as the offset to mmap. 147373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell * See ipath_mmap() for details. 148373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell */ 149373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if (udata && udata->outlen >= sizeof(__u64)) { 150373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell int err; 1516b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh u32 s = sizeof(struct ipath_rwq) + srq->rq.size * sz; 152373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell 1536b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh srq->ip = 1546b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh ipath_create_mmap_info(dev, s, 1556b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh ibpd->uobject->context, 1566b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh srq->rq.wq); 1576b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh if (!srq->ip) { 1586b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh ret = ERR_PTR(-ENOMEM); 159373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell goto bail_wq; 160373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell } 161373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell 1626b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh err = ib_copy_to_udata(udata, &srq->ip->offset, 1636b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh sizeof(srq->ip->offset)); 1646b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh if (err) { 1656b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh ret = ERR_PTR(err); 1666b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh goto bail_ip; 167373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell } 168373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell } else 169373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell srq->ip = NULL; 170373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell 171373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell /* 172cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * ib_create_srq() will initialize srq->ibsrq. 173cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan */ 174cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan spin_lock_init(&srq->rq.lock); 175373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell srq->rq.wq->head = 0; 176373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell srq->rq.wq->tail = 0; 177cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan srq->limit = srq_init_attr->attr.srq_limit; 178cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 179f3e93c7757043cd5d5c4879b8b92effcc7817c81Bryan O'Sullivan spin_lock(&dev->n_srqs_lock); 180f3e93c7757043cd5d5c4879b8b92effcc7817c81Bryan O'Sullivan if (dev->n_srqs_allocated == ib_ipath_max_srqs) { 181f3e93c7757043cd5d5c4879b8b92effcc7817c81Bryan O'Sullivan spin_unlock(&dev->n_srqs_lock); 182f3e93c7757043cd5d5c4879b8b92effcc7817c81Bryan O'Sullivan ret = ERR_PTR(-ENOMEM); 1836b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh goto bail_ip; 184f3e93c7757043cd5d5c4879b8b92effcc7817c81Bryan O'Sullivan } 185f3e93c7757043cd5d5c4879b8b92effcc7817c81Bryan O'Sullivan 186f3e93c7757043cd5d5c4879b8b92effcc7817c81Bryan O'Sullivan dev->n_srqs_allocated++; 187f3e93c7757043cd5d5c4879b8b92effcc7817c81Bryan O'Sullivan spin_unlock(&dev->n_srqs_lock); 188373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell 1896b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh if (srq->ip) { 1906b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh spin_lock_irq(&dev->pending_lock); 1916b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh list_add(&srq->ip->pending_mmaps, &dev->pending_mmaps); 1926b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh spin_unlock_irq(&dev->pending_lock); 1936b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh } 1946b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh 195cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan ret = &srq->ibsrq; 196373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell goto done; 197cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 1986b66b2da1e821181a001c00b04a807724ad803cdRobert Walshbail_ip: 1996b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh kfree(srq->ip); 200373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbellbail_wq: 201373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell vfree(srq->rq.wq); 202373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbellbail_srq: 203373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell kfree(srq); 204373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbelldone: 205cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan return ret; 206cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan} 207cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 208cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan/** 209cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * ipath_modify_srq - modify a shared receive queue 210cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * @ibsrq: the SRQ to modify 211cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * @attr: the new attributes of the SRQ 212cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * @attr_mask: indicates which attributes to modify 2139bc57e2d19db4da81c1150120658cc3658a99ed4Ralph Campbell * @udata: user data for ipathverbs.so 214cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan */ 215cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivanint ipath_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 2169bc57e2d19db4da81c1150120658cc3658a99ed4Ralph Campbell enum ib_srq_attr_mask attr_mask, 2179bc57e2d19db4da81c1150120658cc3658a99ed4Ralph Campbell struct ib_udata *udata) 218cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan{ 219cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan struct ipath_srq *srq = to_isrq(ibsrq); 22014de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell struct ipath_rwq *wq; 221373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell int ret = 0; 222cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 223373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if (attr_mask & IB_SRQ_MAX_WR) { 224373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell struct ipath_rwq *owq; 225373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell struct ipath_rwqe *p; 226373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell u32 sz, size, n, head, tail; 227cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 228373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell /* Check that the requested sizes are below the limits. */ 229373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if ((attr->max_wr > ib_ipath_max_srq_wrs) || 230373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell ((attr_mask & IB_SRQ_LIMIT) ? 231373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell attr->srq_limit : srq->limit) > attr->max_wr) { 232cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan ret = -EINVAL; 233cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan goto bail; 234cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan } 235cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 236cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan sz = sizeof(struct ipath_rwqe) + 237373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell srq->rq.max_sge * sizeof(struct ib_sge); 238fe62546a6afa141c4ab9aef65f5978a1b36cb523Bryan O'Sullivan size = attr->max_wr + 1; 239373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell wq = vmalloc_user(sizeof(struct ipath_rwq) + size * sz); 240cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan if (!wq) { 241cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan ret = -ENOMEM; 242cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan goto bail; 243cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan } 244cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 24514de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell /* Check that we can write the offset to mmap. */ 246373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if (udata && udata->inlen >= sizeof(__u64)) { 247373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell __u64 offset_addr; 24814de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell __u64 offset = 0; 249373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell 250373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell ret = ib_copy_from_udata(&offset_addr, udata, 251373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell sizeof(offset_addr)); 25214de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell if (ret) 25314de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell goto bail_free; 2543dd59e226e01ddb5b041eb0b2e7c7f28b1f730c9Arthur Jones udata->outbuf = 2553dd59e226e01ddb5b041eb0b2e7c7f28b1f730c9Arthur Jones (void __user *) (unsigned long) offset_addr; 256373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell ret = ib_copy_to_udata(udata, &offset, 257373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell sizeof(offset)); 25814de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell if (ret) 25914de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell goto bail_free; 260373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell } 261373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell 262373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell spin_lock_irq(&srq->rq.lock); 263373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell /* 264373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell * validate head pointer value and compute 265373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell * the number of remaining WQEs. 266373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell */ 267373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell owq = srq->rq.wq; 268373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell head = owq->head; 269373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if (head >= srq->rq.size) 270373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell head = 0; 271373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell tail = owq->tail; 272373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if (tail >= srq->rq.size) 273373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell tail = 0; 274373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell n = head; 275373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if (n < tail) 276373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell n += srq->rq.size - tail; 277cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan else 278373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell n -= tail; 279373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if (size <= n) { 280cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan ret = -EINVAL; 28114de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell goto bail_unlock; 282cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan } 283cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan n = 0; 284373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell p = wq->wq; 285373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell while (tail != head) { 286cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan struct ipath_rwqe *wqe; 287cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan int i; 288cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 289373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell wqe = get_rwqe_ptr(&srq->rq, tail); 290cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan p->wr_id = wqe->wr_id; 291cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan p->num_sge = wqe->num_sge; 292cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan for (i = 0; i < wqe->num_sge; i++) 293cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan p->sg_list[i] = wqe->sg_list[i]; 294cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan n++; 295cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan p = (struct ipath_rwqe *)((char *) p + sz); 296373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if (++tail >= srq->rq.size) 297373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell tail = 0; 298cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan } 299cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan srq->rq.wq = wq; 300cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan srq->rq.size = size; 301373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell wq->head = n; 302373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell wq->tail = 0; 303373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if (attr_mask & IB_SRQ_LIMIT) 304373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell srq->limit = attr->srq_limit; 305373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell spin_unlock_irq(&srq->rq.lock); 306373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell 307373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell vfree(owq); 308373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell 309373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if (srq->ip) { 310373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell struct ipath_mmap_info *ip = srq->ip; 311373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell struct ipath_ibdev *dev = to_idev(srq->ibsrq.device); 3126b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh u32 s = sizeof(struct ipath_rwq) + size * sz; 313373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell 3146b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh ipath_update_mmap_info(dev, ip, s, wq); 31514de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell 31614de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell /* 31714de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell * Return the offset to mmap. 31814de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell * See ipath_mmap() for details. 31914de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell */ 32014de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell if (udata && udata->inlen >= sizeof(__u64)) { 32114de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell ret = ib_copy_to_udata(udata, &ip->offset, 32214de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell sizeof(ip->offset)); 32314de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell if (ret) 32414de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell goto bail; 32514de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell } 32614de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell 327373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell spin_lock_irq(&dev->pending_lock); 3286b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh if (list_empty(&ip->pending_mmaps)) 3296b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh list_add(&ip->pending_mmaps, 3306b66b2da1e821181a001c00b04a807724ad803cdRobert Walsh &dev->pending_mmaps); 331373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell spin_unlock_irq(&dev->pending_lock); 332373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell } 333373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell } else if (attr_mask & IB_SRQ_LIMIT) { 334373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell spin_lock_irq(&srq->rq.lock); 335373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if (attr->srq_limit >= srq->rq.size) 336373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell ret = -EINVAL; 337373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell else 338373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell srq->limit = attr->srq_limit; 339373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell spin_unlock_irq(&srq->rq.lock); 340fe62546a6afa141c4ab9aef65f5978a1b36cb523Bryan O'Sullivan } 34114de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell goto bail; 342cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 34314de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbellbail_unlock: 34414de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell spin_unlock_irq(&srq->rq.lock); 34514de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbellbail_free: 34614de986a0ba560b54340fd277a3579e95a2d3838Ralph Campbell vfree(wq); 347cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivanbail: 348cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan return ret; 349cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan} 350cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 351cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivanint ipath_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) 352cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan{ 353cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan struct ipath_srq *srq = to_isrq(ibsrq); 354cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 355cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan attr->max_wr = srq->rq.size - 1; 356cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan attr->max_sge = srq->rq.max_sge; 357cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan attr->srq_limit = srq->limit; 358cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan return 0; 359cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan} 360cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 361cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan/** 362cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * ipath_destroy_srq - destroy a shared receive queue 363cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan * @ibsrq: the SRQ to destroy 364cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan */ 365cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivanint ipath_destroy_srq(struct ib_srq *ibsrq) 366cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan{ 367cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan struct ipath_srq *srq = to_isrq(ibsrq); 368fe62546a6afa141c4ab9aef65f5978a1b36cb523Bryan O'Sullivan struct ipath_ibdev *dev = to_idev(ibsrq->device); 369cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 370f3e93c7757043cd5d5c4879b8b92effcc7817c81Bryan O'Sullivan spin_lock(&dev->n_srqs_lock); 371fe62546a6afa141c4ab9aef65f5978a1b36cb523Bryan O'Sullivan dev->n_srqs_allocated--; 372f3e93c7757043cd5d5c4879b8b92effcc7817c81Bryan O'Sullivan spin_unlock(&dev->n_srqs_lock); 373f3e93c7757043cd5d5c4879b8b92effcc7817c81Bryan O'Sullivan if (srq->ip) 374f3e93c7757043cd5d5c4879b8b92effcc7817c81Bryan O'Sullivan kref_put(&srq->ip->ref, ipath_release_mmap_info); 375f3e93c7757043cd5d5c4879b8b92effcc7817c81Bryan O'Sullivan else 376f3e93c7757043cd5d5c4879b8b92effcc7817c81Bryan O'Sullivan vfree(srq->rq.wq); 377cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan kfree(srq); 378cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan 379cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan return 0; 380cef1cce5c87d84f76e44f0e7b4de72ab3818ac3aBryan O'Sullivan} 381