1225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier/* 2225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. 351a379d0c8f7a6db7c9e3c9c770d90a6d2d1ef9bJack Morgenstein * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * 5225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * This software is available to you under a choice of one of two 6225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * licenses. You may choose to be licensed under the terms of the GNU 7225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * General Public License (GPL) Version 2, available from the file 8225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * COPYING in the main directory of this source tree, or the 9225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * OpenIB.org BSD license below: 10225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * 11225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * Redistribution and use in source and binary forms, with or 12225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * without modification, are permitted provided that the following 13225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * conditions are met: 14225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * 15225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * - Redistributions of source code must retain the above 16225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * copyright notice, this list of conditions and the following 17225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * disclaimer. 18225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * 19225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * - Redistributions in binary form must reproduce the above 20225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * copyright notice, this list of conditions and the following 21225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * disclaimer in the documentation and/or other materials 22225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * provided with the distribution. 23225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * 24225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * SOFTWARE. 32225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier */ 33225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 34225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier#include <linux/mlx4/qp.h> 35225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier#include <linux/mlx4/srq.h> 365a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 37225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 38225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier#include "mlx4_ib.h" 39225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier#include "user.h" 40225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 41225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreierstatic void *get_wqe(struct mlx4_ib_srq *srq, int n) 42225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier{ 431c69fc2a9012e160c8d459f63df74a6b01db8322Roland Dreier return mlx4_buf_offset(&srq->buf, n << srq->msrq.wqe_shift); 44225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier} 45225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 46225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreierstatic void mlx4_ib_srq_event(struct mlx4_srq *srq, enum mlx4_event type) 47225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier{ 48225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier struct ib_event event; 49225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq; 50225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 51225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (ibsrq->event_handler) { 52225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier event.device = ibsrq->device; 53225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier event.element.srq = ibsrq; 54225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier switch (type) { 55225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier case MLX4_EVENT_TYPE_SRQ_LIMIT: 56225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier event.event = IB_EVENT_SRQ_LIMIT_REACHED; 57225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier break; 58225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR: 59225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier event.event = IB_EVENT_SRQ_ERR; 60225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier break; 61225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier default: 62225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier printk(KERN_WARNING "mlx4_ib: Unexpected event type %d " 63225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier "on SRQ %06x\n", type, srq->srqn); 64225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier return; 65225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier } 66225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 67225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier ibsrq->event_handler(&event, ibsrq->srq_context); 68225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier } 69225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier} 70225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 71225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreierstruct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd, 72225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier struct ib_srq_init_attr *init_attr, 73225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier struct ib_udata *udata) 74225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier{ 75225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier struct mlx4_ib_dev *dev = to_mdev(pd->device); 76225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier struct mlx4_ib_srq *srq; 77225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier struct mlx4_wqe_srq_next_seg *next; 784c425588e0d72c9c55024752b0f6e709c96787ffJack Morgenstein struct mlx4_wqe_data_seg *scatter; 7918abd5ea571608a7c726fc56e21d3e31f9febfd0Sean Hefty u32 cqn; 8018abd5ea571608a7c726fc56e21d3e31f9febfd0Sean Hefty u16 xrcdn; 81225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier int desc_size; 82225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier int buf_size; 83225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier int err; 84225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier int i; 85225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 86225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier /* Sanity check SRQ size before proceeding */ 87225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (init_attr->attr.max_wr >= dev->dev->caps.max_srq_wqes || 88225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier init_attr->attr.max_sge > dev->dev->caps.max_srq_sge) 89225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier return ERR_PTR(-EINVAL); 90225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 91225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier srq = kmalloc(sizeof *srq, GFP_KERNEL); 92225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (!srq) 93225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier return ERR_PTR(-ENOMEM); 94225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 95225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier mutex_init(&srq->mutex); 96225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier spin_lock_init(&srq->lock); 97225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1); 98225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier srq->msrq.max_gs = init_attr->attr.max_sge; 99225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 100225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier desc_size = max(32UL, 101225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier roundup_pow_of_two(sizeof (struct mlx4_wqe_srq_next_seg) + 102225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier srq->msrq.max_gs * 103225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier sizeof (struct mlx4_wqe_data_seg))); 104225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier srq->msrq.wqe_shift = ilog2(desc_size); 105225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 106225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier buf_size = srq->msrq.max * desc_size; 107225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 108225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (pd->uobject) { 109225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier struct mlx4_ib_create_srq ucmd; 110225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 111225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { 112225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier err = -EFAULT; 113225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier goto err_srq; 114225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier } 115225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 116225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier srq->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr, 117cb9fbc5c37b69ac584e61d449cfd590f5ae1f90dArthur Kepner buf_size, 0, 0); 118225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (IS_ERR(srq->umem)) { 119225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier err = PTR_ERR(srq->umem); 120225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier goto err_srq; 121225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier } 122225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 123225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier err = mlx4_mtt_init(dev->dev, ib_umem_page_count(srq->umem), 124225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier ilog2(srq->umem->page_size), &srq->mtt); 125225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (err) 126225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier goto err_buf; 127225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 128225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier err = mlx4_ib_umem_write_mtt(dev, &srq->mtt, srq->umem); 129225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (err) 130225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier goto err_mtt; 131225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 132225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context), 133225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier ucmd.db_addr, &srq->db); 134225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (err) 135225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier goto err_mtt; 136225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier } else { 1376296883ca4cd52dafb45f191d24102e28ded38f2Yevgeny Petrilin err = mlx4_db_alloc(dev->dev, &srq->db, 0); 138225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (err) 139225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier goto err_srq; 140225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 141225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier *srq->db.db = 0; 142225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 143225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (mlx4_buf_alloc(dev->dev, buf_size, PAGE_SIZE * 2, &srq->buf)) { 144225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier err = -ENOMEM; 145225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier goto err_db; 146225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier } 147225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 148225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier srq->head = 0; 149225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier srq->tail = srq->msrq.max - 1; 150225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier srq->wqe_ctr = 0; 151225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 152225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier for (i = 0; i < srq->msrq.max; ++i) { 153225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier next = get_wqe(srq, i); 154225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier next->next_wqe_index = 155225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier cpu_to_be16((i + 1) & (srq->msrq.max - 1)); 1564c425588e0d72c9c55024752b0f6e709c96787ffJack Morgenstein 1574c425588e0d72c9c55024752b0f6e709c96787ffJack Morgenstein for (scatter = (void *) (next + 1); 1584c425588e0d72c9c55024752b0f6e709c96787ffJack Morgenstein (void *) scatter < (void *) next + desc_size; 1594c425588e0d72c9c55024752b0f6e709c96787ffJack Morgenstein ++scatter) 1604c425588e0d72c9c55024752b0f6e709c96787ffJack Morgenstein scatter->lkey = cpu_to_be32(MLX4_INVALID_LKEY); 161225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier } 162225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 163225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier err = mlx4_mtt_init(dev->dev, srq->buf.npages, srq->buf.page_shift, 164225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier &srq->mtt); 165225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (err) 166225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier goto err_buf; 167225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 168225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier err = mlx4_buf_write_mtt(dev->dev, &srq->mtt, &srq->buf); 169225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (err) 170225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier goto err_mtt; 171225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 172225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier srq->wrid = kmalloc(srq->msrq.max * sizeof (u64), GFP_KERNEL); 173225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (!srq->wrid) { 174225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier err = -ENOMEM; 175225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier goto err_mtt; 176225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier } 177225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier } 178225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 17918abd5ea571608a7c726fc56e21d3e31f9febfd0Sean Hefty cqn = (init_attr->srq_type == IB_SRQT_XRC) ? 18018abd5ea571608a7c726fc56e21d3e31f9febfd0Sean Hefty to_mcq(init_attr->ext.xrc.cq)->mcq.cqn : 0; 18118abd5ea571608a7c726fc56e21d3e31f9febfd0Sean Hefty xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ? 18218abd5ea571608a7c726fc56e21d3e31f9febfd0Sean Hefty to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn : 18318abd5ea571608a7c726fc56e21d3e31f9febfd0Sean Hefty (u16) dev->dev->caps.reserved_xrcds; 18418abd5ea571608a7c726fc56e21d3e31f9febfd0Sean Hefty err = mlx4_srq_alloc(dev->dev, to_mpd(pd)->pdn, cqn, xrcdn, &srq->mtt, 185225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier srq->db.dma, &srq->msrq); 186225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (err) 187225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier goto err_wrid; 188225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 189225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier srq->msrq.event = mlx4_ib_srq_event; 19018abd5ea571608a7c726fc56e21d3e31f9febfd0Sean Hefty srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn; 191225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 192225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (pd->uobject) 193225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) { 194225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier err = -EFAULT; 195225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier goto err_wrid; 196225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier } 197225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 198225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier init_attr->attr.max_wr = srq->msrq.max - 1; 199225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 200225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier return &srq->ibsrq; 201225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 202225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreiererr_wrid: 203225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (pd->uobject) 204225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &srq->db); 205225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier else 206225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier kfree(srq->wrid); 207225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 208225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreiererr_mtt: 209225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier mlx4_mtt_cleanup(dev->dev, &srq->mtt); 210225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 211225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreiererr_buf: 212225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (pd->uobject) 213225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier ib_umem_release(srq->umem); 214225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier else 215225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier mlx4_buf_free(dev->dev, buf_size, &srq->buf); 216225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 217225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreiererr_db: 218225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (!pd->uobject) 2196296883ca4cd52dafb45f191d24102e28ded38f2Yevgeny Petrilin mlx4_db_free(dev->dev, &srq->db); 220225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 221225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreiererr_srq: 222225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier kfree(srq); 223225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 224225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier return ERR_PTR(err); 225225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier} 226225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 227225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreierint mlx4_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 228225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 229225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier{ 230225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier struct mlx4_ib_dev *dev = to_mdev(ibsrq->device); 231225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier struct mlx4_ib_srq *srq = to_msrq(ibsrq); 232225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier int ret; 233225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 234225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier /* We don't support resizing SRQs (yet?) */ 235225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (attr_mask & IB_SRQ_MAX_WR) 236225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier return -EINVAL; 237225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 238225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (attr_mask & IB_SRQ_LIMIT) { 239225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (attr->srq_limit >= srq->msrq.max) 240225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier return -EINVAL; 241225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 242225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier mutex_lock(&srq->mutex); 243225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier ret = mlx4_srq_arm(dev->dev, &srq->msrq, attr->srq_limit); 244225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier mutex_unlock(&srq->mutex); 245225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 246225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (ret) 247225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier return ret; 248225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier } 249225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 250225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier return 0; 251225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier} 252225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 25365541cb7cf353946ecd78016a453b453b8830656Jack Morgensteinint mlx4_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 25465541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein{ 25565541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein struct mlx4_ib_dev *dev = to_mdev(ibsrq->device); 25665541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein struct mlx4_ib_srq *srq = to_msrq(ibsrq); 25765541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein int ret; 25865541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein int limit_watermark; 25965541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein 26065541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein ret = mlx4_srq_query(dev->dev, &srq->msrq, &limit_watermark); 26165541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein if (ret) 26265541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein return ret; 26365541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein 264d7dc3ccbe4dd1d37950da0138079e61d5be06ca9Roland Dreier srq_attr->srq_limit = limit_watermark; 26565541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein srq_attr->max_wr = srq->msrq.max - 1; 26665541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein srq_attr->max_sge = srq->msrq.max_gs; 26765541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein 26865541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein return 0; 26965541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein} 27065541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein 271225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreierint mlx4_ib_destroy_srq(struct ib_srq *srq) 272225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier{ 273225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier struct mlx4_ib_dev *dev = to_mdev(srq->device); 274225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier struct mlx4_ib_srq *msrq = to_msrq(srq); 275225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 276225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier mlx4_srq_free(dev->dev, &msrq->msrq); 277225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier mlx4_mtt_cleanup(dev->dev, &msrq->mtt); 278225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 279225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (srq->uobject) { 280225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier mlx4_ib_db_unmap_user(to_mucontext(srq->uobject->context), &msrq->db); 281225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier ib_umem_release(msrq->umem); 282225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier } else { 283225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier kfree(msrq->wrid); 284225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier mlx4_buf_free(dev->dev, msrq->msrq.max << msrq->msrq.wqe_shift, 285225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier &msrq->buf); 2866296883ca4cd52dafb45f191d24102e28ded38f2Yevgeny Petrilin mlx4_db_free(dev->dev, &msrq->db); 287225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier } 288225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 289225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier kfree(msrq); 290225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 291225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier return 0; 292225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier} 293225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 294225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreiervoid mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index) 295225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier{ 296225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier struct mlx4_wqe_srq_next_seg *next; 297225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 298225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier /* always called with interrupts disabled. */ 299225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier spin_lock(&srq->lock); 300225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 301225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier next = get_wqe(srq, srq->tail); 302225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier next->next_wqe_index = cpu_to_be16(wqe_index); 303225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier srq->tail = wqe_index; 304225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 305225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier spin_unlock(&srq->lock); 306225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier} 307225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 308225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreierint mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, 309225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier struct ib_recv_wr **bad_wr) 310225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier{ 311225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier struct mlx4_ib_srq *srq = to_msrq(ibsrq); 312225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier struct mlx4_wqe_srq_next_seg *next; 313225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier struct mlx4_wqe_data_seg *scat; 314225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier unsigned long flags; 315225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier int err = 0; 316225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier int nreq; 317225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier int i; 318225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 319225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier spin_lock_irqsave(&srq->lock, flags); 320225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 321225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier for (nreq = 0; wr; ++nreq, wr = wr->next) { 322225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (unlikely(wr->num_sge > srq->msrq.max_gs)) { 323225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier err = -EINVAL; 324225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier *bad_wr = wr; 325225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier break; 326225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier } 327225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 32856a8c8b6ac4d6edba5153d17730aaf96ba8f1f8cRoland Dreier if (unlikely(srq->head == srq->tail)) { 32956a8c8b6ac4d6edba5153d17730aaf96ba8f1f8cRoland Dreier err = -ENOMEM; 33056a8c8b6ac4d6edba5153d17730aaf96ba8f1f8cRoland Dreier *bad_wr = wr; 33156a8c8b6ac4d6edba5153d17730aaf96ba8f1f8cRoland Dreier break; 33256a8c8b6ac4d6edba5153d17730aaf96ba8f1f8cRoland Dreier } 33356a8c8b6ac4d6edba5153d17730aaf96ba8f1f8cRoland Dreier 334225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier srq->wrid[srq->head] = wr->wr_id; 335225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 336225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier next = get_wqe(srq, srq->head); 337225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier srq->head = be16_to_cpu(next->next_wqe_index); 338225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier scat = (struct mlx4_wqe_data_seg *) (next + 1); 339225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 340225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier for (i = 0; i < wr->num_sge; ++i) { 341225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier scat[i].byte_count = cpu_to_be32(wr->sg_list[i].length); 342225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier scat[i].lkey = cpu_to_be32(wr->sg_list[i].lkey); 343225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier scat[i].addr = cpu_to_be64(wr->sg_list[i].addr); 344225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier } 345225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 346225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (i < srq->msrq.max_gs) { 347225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier scat[i].byte_count = 0; 348225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY); 349225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier scat[i].addr = 0; 350225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier } 351225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier } 352225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 353225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier if (likely(nreq)) { 354225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier srq->wqe_ctr += nreq; 355225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 356225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier /* 357225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * Make sure that descriptors are written before 358225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * doorbell record. 359225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier */ 360225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier wmb(); 361225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 362225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier *srq->db.db = cpu_to_be32(srq->wqe_ctr); 363225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier } 364225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 365225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier spin_unlock_irqrestore(&srq->lock, flags); 366225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier 367225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier return err; 368225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier} 369