1225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier/*
2225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
351a379d0c8f7a6db7c9e3c9c770d90a6d2d1ef9bJack Morgenstein * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
4225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier *
5225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * This software is available to you under a choice of one of two
6225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * licenses.  You may choose to be licensed under the terms of the GNU
7225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * General Public License (GPL) Version 2, available from the file
8225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * COPYING in the main directory of this source tree, or the
9225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * OpenIB.org BSD license below:
10225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier *
11225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier *     Redistribution and use in source and binary forms, with or
12225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier *     without modification, are permitted provided that the following
13225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier *     conditions are met:
14225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier *
15225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier *      - Redistributions of source code must retain the above
16225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier *        copyright notice, this list of conditions and the following
17225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier *        disclaimer.
18225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier *
19225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier *      - Redistributions in binary form must reproduce the above
20225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier *        copyright notice, this list of conditions and the following
21225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier *        disclaimer in the documentation and/or other materials
22225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier *        provided with the distribution.
23225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier *
24225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier * SOFTWARE.
32225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier */
33225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
34225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier#include <linux/mlx4/qp.h>
35225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier#include <linux/mlx4/srq.h>
365a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h>
37225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
38225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier#include "mlx4_ib.h"
39225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier#include "user.h"
40225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
41225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreierstatic void *get_wqe(struct mlx4_ib_srq *srq, int n)
42225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier{
431c69fc2a9012e160c8d459f63df74a6b01db8322Roland Dreier	return mlx4_buf_offset(&srq->buf, n << srq->msrq.wqe_shift);
44225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier}
45225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
46225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreierstatic void mlx4_ib_srq_event(struct mlx4_srq *srq, enum mlx4_event type)
47225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier{
48225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	struct ib_event event;
49225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq;
50225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
51225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	if (ibsrq->event_handler) {
52225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		event.device      = ibsrq->device;
53225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		event.element.srq = ibsrq;
54225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		switch (type) {
55225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		case MLX4_EVENT_TYPE_SRQ_LIMIT:
56225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			event.event = IB_EVENT_SRQ_LIMIT_REACHED;
57225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			break;
58225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR:
59225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			event.event = IB_EVENT_SRQ_ERR;
60225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			break;
61225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		default:
62225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			printk(KERN_WARNING "mlx4_ib: Unexpected event type %d "
63225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			       "on SRQ %06x\n", type, srq->srqn);
64225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			return;
65225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		}
66225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
67225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		ibsrq->event_handler(&event, ibsrq->srq_context);
68225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	}
69225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier}
70225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
71225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreierstruct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd,
72225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier				  struct ib_srq_init_attr *init_attr,
73225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier				  struct ib_udata *udata)
74225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier{
75225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	struct mlx4_ib_dev *dev = to_mdev(pd->device);
76225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	struct mlx4_ib_srq *srq;
77225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	struct mlx4_wqe_srq_next_seg *next;
784c425588e0d72c9c55024752b0f6e709c96787ffJack Morgenstein	struct mlx4_wqe_data_seg *scatter;
7918abd5ea571608a7c726fc56e21d3e31f9febfd0Sean Hefty	u32 cqn;
8018abd5ea571608a7c726fc56e21d3e31f9febfd0Sean Hefty	u16 xrcdn;
81225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	int desc_size;
82225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	int buf_size;
83225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	int err;
84225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	int i;
85225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
86225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	/* Sanity check SRQ size before proceeding */
87225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	if (init_attr->attr.max_wr  >= dev->dev->caps.max_srq_wqes ||
88225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	    init_attr->attr.max_sge >  dev->dev->caps.max_srq_sge)
89225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		return ERR_PTR(-EINVAL);
90225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
91225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	srq = kmalloc(sizeof *srq, GFP_KERNEL);
92225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	if (!srq)
93225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		return ERR_PTR(-ENOMEM);
94225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
95225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	mutex_init(&srq->mutex);
96225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	spin_lock_init(&srq->lock);
97225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	srq->msrq.max    = roundup_pow_of_two(init_attr->attr.max_wr + 1);
98225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	srq->msrq.max_gs = init_attr->attr.max_sge;
99225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
100225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	desc_size = max(32UL,
101225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			roundup_pow_of_two(sizeof (struct mlx4_wqe_srq_next_seg) +
102225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier					   srq->msrq.max_gs *
103225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier					   sizeof (struct mlx4_wqe_data_seg)));
104225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	srq->msrq.wqe_shift = ilog2(desc_size);
105225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
106225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	buf_size = srq->msrq.max * desc_size;
107225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
108225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	if (pd->uobject) {
109225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		struct mlx4_ib_create_srq ucmd;
110225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
111225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
112225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			err = -EFAULT;
113225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			goto err_srq;
114225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		}
115225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
116225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		srq->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr,
117cb9fbc5c37b69ac584e61d449cfd590f5ae1f90dArthur Kepner					buf_size, 0, 0);
118225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		if (IS_ERR(srq->umem)) {
119225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			err = PTR_ERR(srq->umem);
120225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			goto err_srq;
121225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		}
122225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
123225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		err = mlx4_mtt_init(dev->dev, ib_umem_page_count(srq->umem),
124225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier				    ilog2(srq->umem->page_size), &srq->mtt);
125225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		if (err)
126225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			goto err_buf;
127225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
128225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		err = mlx4_ib_umem_write_mtt(dev, &srq->mtt, srq->umem);
129225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		if (err)
130225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			goto err_mtt;
131225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
132225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context),
133225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier					  ucmd.db_addr, &srq->db);
134225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		if (err)
135225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			goto err_mtt;
136225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	} else {
1376296883ca4cd52dafb45f191d24102e28ded38f2Yevgeny Petrilin		err = mlx4_db_alloc(dev->dev, &srq->db, 0);
138225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		if (err)
139225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			goto err_srq;
140225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
141225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		*srq->db.db = 0;
142225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
143225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		if (mlx4_buf_alloc(dev->dev, buf_size, PAGE_SIZE * 2, &srq->buf)) {
144225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			err = -ENOMEM;
145225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			goto err_db;
146225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		}
147225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
148225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		srq->head    = 0;
149225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		srq->tail    = srq->msrq.max - 1;
150225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		srq->wqe_ctr = 0;
151225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
152225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		for (i = 0; i < srq->msrq.max; ++i) {
153225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			next = get_wqe(srq, i);
154225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			next->next_wqe_index =
155225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier				cpu_to_be16((i + 1) & (srq->msrq.max - 1));
1564c425588e0d72c9c55024752b0f6e709c96787ffJack Morgenstein
1574c425588e0d72c9c55024752b0f6e709c96787ffJack Morgenstein			for (scatter = (void *) (next + 1);
1584c425588e0d72c9c55024752b0f6e709c96787ffJack Morgenstein			     (void *) scatter < (void *) next + desc_size;
1594c425588e0d72c9c55024752b0f6e709c96787ffJack Morgenstein			     ++scatter)
1604c425588e0d72c9c55024752b0f6e709c96787ffJack Morgenstein				scatter->lkey = cpu_to_be32(MLX4_INVALID_LKEY);
161225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		}
162225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
163225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		err = mlx4_mtt_init(dev->dev, srq->buf.npages, srq->buf.page_shift,
164225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier				    &srq->mtt);
165225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		if (err)
166225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			goto err_buf;
167225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
168225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		err = mlx4_buf_write_mtt(dev->dev, &srq->mtt, &srq->buf);
169225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		if (err)
170225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			goto err_mtt;
171225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
172225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		srq->wrid = kmalloc(srq->msrq.max * sizeof (u64), GFP_KERNEL);
173225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		if (!srq->wrid) {
174225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			err = -ENOMEM;
175225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			goto err_mtt;
176225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		}
177225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	}
178225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
17918abd5ea571608a7c726fc56e21d3e31f9febfd0Sean Hefty	cqn = (init_attr->srq_type == IB_SRQT_XRC) ?
18018abd5ea571608a7c726fc56e21d3e31f9febfd0Sean Hefty		to_mcq(init_attr->ext.xrc.cq)->mcq.cqn : 0;
18118abd5ea571608a7c726fc56e21d3e31f9febfd0Sean Hefty	xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ?
18218abd5ea571608a7c726fc56e21d3e31f9febfd0Sean Hefty		to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn :
18318abd5ea571608a7c726fc56e21d3e31f9febfd0Sean Hefty		(u16) dev->dev->caps.reserved_xrcds;
18418abd5ea571608a7c726fc56e21d3e31f9febfd0Sean Hefty	err = mlx4_srq_alloc(dev->dev, to_mpd(pd)->pdn, cqn, xrcdn, &srq->mtt,
185225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			     srq->db.dma, &srq->msrq);
186225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	if (err)
187225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		goto err_wrid;
188225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
189225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	srq->msrq.event = mlx4_ib_srq_event;
19018abd5ea571608a7c726fc56e21d3e31f9febfd0Sean Hefty	srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn;
191225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
192225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	if (pd->uobject)
193225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) {
194225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			err = -EFAULT;
195225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			goto err_wrid;
196225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		}
197225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
198225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	init_attr->attr.max_wr = srq->msrq.max - 1;
199225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
200225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	return &srq->ibsrq;
201225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
202225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreiererr_wrid:
203225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	if (pd->uobject)
204225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &srq->db);
205225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	else
206225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		kfree(srq->wrid);
207225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
208225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreiererr_mtt:
209225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	mlx4_mtt_cleanup(dev->dev, &srq->mtt);
210225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
211225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreiererr_buf:
212225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	if (pd->uobject)
213225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		ib_umem_release(srq->umem);
214225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	else
215225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		mlx4_buf_free(dev->dev, buf_size, &srq->buf);
216225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
217225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreiererr_db:
218225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	if (!pd->uobject)
2196296883ca4cd52dafb45f191d24102e28ded38f2Yevgeny Petrilin		mlx4_db_free(dev->dev, &srq->db);
220225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
221225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreiererr_srq:
222225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	kfree(srq);
223225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
224225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	return ERR_PTR(err);
225225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier}
226225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
227225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreierint mlx4_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
228225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		       enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
229225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier{
230225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	struct mlx4_ib_dev *dev = to_mdev(ibsrq->device);
231225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	struct mlx4_ib_srq *srq = to_msrq(ibsrq);
232225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	int ret;
233225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
234225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	/* We don't support resizing SRQs (yet?) */
235225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	if (attr_mask & IB_SRQ_MAX_WR)
236225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		return -EINVAL;
237225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
238225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	if (attr_mask & IB_SRQ_LIMIT) {
239225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		if (attr->srq_limit >= srq->msrq.max)
240225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			return -EINVAL;
241225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
242225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		mutex_lock(&srq->mutex);
243225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		ret = mlx4_srq_arm(dev->dev, &srq->msrq, attr->srq_limit);
244225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		mutex_unlock(&srq->mutex);
245225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
246225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		if (ret)
247225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			return ret;
248225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	}
249225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
250225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	return 0;
251225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier}
252225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
25365541cb7cf353946ecd78016a453b453b8830656Jack Morgensteinint mlx4_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
25465541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein{
25565541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein	struct mlx4_ib_dev *dev = to_mdev(ibsrq->device);
25665541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein	struct mlx4_ib_srq *srq = to_msrq(ibsrq);
25765541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein	int ret;
25865541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein	int limit_watermark;
25965541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein
26065541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein	ret = mlx4_srq_query(dev->dev, &srq->msrq, &limit_watermark);
26165541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein	if (ret)
26265541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein		return ret;
26365541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein
264d7dc3ccbe4dd1d37950da0138079e61d5be06ca9Roland Dreier	srq_attr->srq_limit = limit_watermark;
26565541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein	srq_attr->max_wr    = srq->msrq.max - 1;
26665541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein	srq_attr->max_sge   = srq->msrq.max_gs;
26765541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein
26865541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein	return 0;
26965541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein}
27065541cb7cf353946ecd78016a453b453b8830656Jack Morgenstein
271225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreierint mlx4_ib_destroy_srq(struct ib_srq *srq)
272225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier{
273225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	struct mlx4_ib_dev *dev = to_mdev(srq->device);
274225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	struct mlx4_ib_srq *msrq = to_msrq(srq);
275225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
276225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	mlx4_srq_free(dev->dev, &msrq->msrq);
277225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	mlx4_mtt_cleanup(dev->dev, &msrq->mtt);
278225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
279225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	if (srq->uobject) {
280225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		mlx4_ib_db_unmap_user(to_mucontext(srq->uobject->context), &msrq->db);
281225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		ib_umem_release(msrq->umem);
282225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	} else {
283225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		kfree(msrq->wrid);
284225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		mlx4_buf_free(dev->dev, msrq->msrq.max << msrq->msrq.wqe_shift,
285225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			      &msrq->buf);
2866296883ca4cd52dafb45f191d24102e28ded38f2Yevgeny Petrilin		mlx4_db_free(dev->dev, &msrq->db);
287225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	}
288225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
289225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	kfree(msrq);
290225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
291225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	return 0;
292225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier}
293225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
294225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreiervoid mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index)
295225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier{
296225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	struct mlx4_wqe_srq_next_seg *next;
297225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
298225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	/* always called with interrupts disabled. */
299225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	spin_lock(&srq->lock);
300225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
301225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	next = get_wqe(srq, srq->tail);
302225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	next->next_wqe_index = cpu_to_be16(wqe_index);
303225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	srq->tail = wqe_index;
304225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
305225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	spin_unlock(&srq->lock);
306225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier}
307225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
308225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreierint mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
309225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			  struct ib_recv_wr **bad_wr)
310225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier{
311225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	struct mlx4_ib_srq *srq = to_msrq(ibsrq);
312225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	struct mlx4_wqe_srq_next_seg *next;
313225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	struct mlx4_wqe_data_seg *scat;
314225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	unsigned long flags;
315225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	int err = 0;
316225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	int nreq;
317225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	int i;
318225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
319225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	spin_lock_irqsave(&srq->lock, flags);
320225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
321225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	for (nreq = 0; wr; ++nreq, wr = wr->next) {
322225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		if (unlikely(wr->num_sge > srq->msrq.max_gs)) {
323225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			err = -EINVAL;
324225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			*bad_wr = wr;
325225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			break;
326225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		}
327225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
32856a8c8b6ac4d6edba5153d17730aaf96ba8f1f8cRoland Dreier		if (unlikely(srq->head == srq->tail)) {
32956a8c8b6ac4d6edba5153d17730aaf96ba8f1f8cRoland Dreier			err = -ENOMEM;
33056a8c8b6ac4d6edba5153d17730aaf96ba8f1f8cRoland Dreier			*bad_wr = wr;
33156a8c8b6ac4d6edba5153d17730aaf96ba8f1f8cRoland Dreier			break;
33256a8c8b6ac4d6edba5153d17730aaf96ba8f1f8cRoland Dreier		}
33356a8c8b6ac4d6edba5153d17730aaf96ba8f1f8cRoland Dreier
334225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		srq->wrid[srq->head] = wr->wr_id;
335225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
336225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		next      = get_wqe(srq, srq->head);
337225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		srq->head = be16_to_cpu(next->next_wqe_index);
338225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		scat      = (struct mlx4_wqe_data_seg *) (next + 1);
339225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
340225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		for (i = 0; i < wr->num_sge; ++i) {
341225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			scat[i].byte_count = cpu_to_be32(wr->sg_list[i].length);
342225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			scat[i].lkey       = cpu_to_be32(wr->sg_list[i].lkey);
343225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			scat[i].addr       = cpu_to_be64(wr->sg_list[i].addr);
344225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		}
345225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
346225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		if (i < srq->msrq.max_gs) {
347225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			scat[i].byte_count = 0;
348225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			scat[i].lkey       = cpu_to_be32(MLX4_INVALID_LKEY);
349225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier			scat[i].addr       = 0;
350225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		}
351225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	}
352225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
353225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	if (likely(nreq)) {
354225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		srq->wqe_ctr += nreq;
355225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
356225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		/*
357225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		 * Make sure that descriptors are written before
358225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		 * doorbell record.
359225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		 */
360225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		wmb();
361225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
362225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier		*srq->db.db = cpu_to_be32(srq->wqe_ctr);
363225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	}
364225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
365225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	spin_unlock_irqrestore(&srq->lock, flags);
366225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier
367225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier	return err;
368225c7b1feef1b41170f7037a5b10a65cd8a42c54Roland Dreier}
369