1f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker/* 2f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Copyright (c) 2004 Topspin Communications. All rights reserved. 3f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Copyright (c) 2005 Cisco Systems. All rights reserved. 4f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Copyright (c) 2005 Mellanox Technologies. All rights reserved. 5f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Copyright (c) 2004 Voltaire, Inc. All rights reserved. 6f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved. 7f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * 8f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * This software is available to you under a choice of one of two 9f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * licenses. You may choose to be licensed under the terms of the GNU 10f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * General Public License (GPL) Version 2, available from the file 11f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * COPYING in the main directory of this source tree, or the 12f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * OpenIB.org BSD license below: 13f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * 14f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Redistribution and use in source and binary forms, with or 15f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * without modification, are permitted provided that the following 16f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * conditions are met: 17f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * 18f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * - Redistributions of source code must retain the above 19f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * copyright notice, this list of conditions and the following 20f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * disclaimer. 21f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * 22f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * - Redistributions in binary form must reproduce the above 23f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * copyright notice, this list of conditions and the following 24f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * disclaimer in the documentation and/or other materials 25f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * provided with the distribution. 26f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * 27f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 28f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 29f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 30f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 31f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 32f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 33f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 34f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * SOFTWARE. 35f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * 36f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 37f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 38e52e6080ca10e0a8ce2a35c86965945cdfa8ed7eTom Tucker#include <linux/delay.h> 395a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/gfp.h> 40e52e6080ca10e0a8ce2a35c86965945cdfa8ed7eTom Tucker 41f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker#include "c2.h" 42f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker#include "c2_vq.h" 43f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker#include "c2_status.h" 44f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 45f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker#define C2_MAX_ORD_PER_QP 128 46f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker#define C2_MAX_IRD_PER_QP 128 47f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 48f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker#define C2_HINT_MAKE(q_index, hint_count) (((q_index) << 16) | hint_count) 49f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker#define C2_HINT_GET_INDEX(hint) (((hint) & 0x7FFF0000) >> 16) 50f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker#define C2_HINT_GET_COUNT(hint) ((hint) & 0x0000FFFF) 51f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 52f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker#define NO_SUPPORT -1 53f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tuckerstatic const u8 c2_opcode[] = { 54f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker [IB_WR_SEND] = C2_WR_TYPE_SEND, 55f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker [IB_WR_SEND_WITH_IMM] = NO_SUPPORT, 56f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker [IB_WR_RDMA_WRITE] = C2_WR_TYPE_RDMA_WRITE, 57f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker [IB_WR_RDMA_WRITE_WITH_IMM] = NO_SUPPORT, 58f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker [IB_WR_RDMA_READ] = C2_WR_TYPE_RDMA_READ, 59f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker [IB_WR_ATOMIC_CMP_AND_SWP] = NO_SUPPORT, 60f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker [IB_WR_ATOMIC_FETCH_AND_ADD] = NO_SUPPORT, 61f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker}; 62f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 63f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tuckerstatic int to_c2_state(enum ib_qp_state ib_state) 64f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker{ 65f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker switch (ib_state) { 66f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker case IB_QPS_RESET: 67f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return C2_QP_STATE_IDLE; 68f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker case IB_QPS_RTS: 69f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return C2_QP_STATE_RTS; 70f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker case IB_QPS_SQD: 71f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return C2_QP_STATE_CLOSING; 72f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker case IB_QPS_SQE: 73f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return C2_QP_STATE_CLOSING; 74f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker case IB_QPS_ERR: 75f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return C2_QP_STATE_ERROR; 76f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker default: 77f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return -1; 78f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 79f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker} 80f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 81f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tuckerstatic int to_ib_state(enum c2_qp_state c2_state) 82f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker{ 83f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker switch (c2_state) { 84f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker case C2_QP_STATE_IDLE: 85f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return IB_QPS_RESET; 86f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker case C2_QP_STATE_CONNECTING: 87f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return IB_QPS_RTR; 88f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker case C2_QP_STATE_RTS: 89f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return IB_QPS_RTS; 90f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker case C2_QP_STATE_CLOSING: 91f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return IB_QPS_SQD; 92f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker case C2_QP_STATE_ERROR: 93f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return IB_QPS_ERR; 94f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker case C2_QP_STATE_TERMINATE: 95f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return IB_QPS_SQE; 96f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker default: 97f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return -1; 98f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 99f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker} 100f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 101f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tuckerstatic const char *to_ib_state_str(int ib_state) 102f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker{ 103f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker static const char *state_str[] = { 104f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker "IB_QPS_RESET", 105f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker "IB_QPS_INIT", 106f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker "IB_QPS_RTR", 107f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker "IB_QPS_RTS", 108f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker "IB_QPS_SQD", 109f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker "IB_QPS_SQE", 110f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker "IB_QPS_ERR" 111f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker }; 112f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (ib_state < IB_QPS_RESET || 113f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker ib_state > IB_QPS_ERR) 114f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return "<invalid IB QP state>"; 115f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 116f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker ib_state -= IB_QPS_RESET; 117f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return state_str[ib_state]; 118f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker} 119f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 120f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tuckervoid c2_set_qp_state(struct c2_qp *qp, int c2_state) 121f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker{ 122f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker int new_state = to_ib_state(c2_state); 123f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 124f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker pr_debug("%s: qp[%p] state modify %s --> %s\n", 1253371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison __func__, 126f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker qp, 127f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker to_ib_state_str(qp->state), 128f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker to_ib_state_str(new_state)); 129f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker qp->state = new_state; 130f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker} 131f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 132f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker#define C2_QP_NO_ATTR_CHANGE 0xFFFFFFFF 133f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 134f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tuckerint c2_qp_modify(struct c2_dev *c2dev, struct c2_qp *qp, 135f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct ib_qp_attr *attr, int attr_mask) 136f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker{ 137f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2wr_qp_modify_req wr; 138f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2wr_qp_modify_rep *reply; 139f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2_vq_req *vq_req; 140f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker unsigned long flags; 141f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker u8 next_state; 142f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker int err; 143f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 144f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker pr_debug("%s:%d qp=%p, %s --> %s\n", 1453371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison __func__, __LINE__, 146f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker qp, 147f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker to_ib_state_str(qp->state), 148f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker to_ib_state_str(attr->qp_state)); 149f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 150f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req = vq_req_alloc(c2dev); 151f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (!vq_req) 152f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return -ENOMEM; 153f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 154f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_wr_set_id(&wr, CCWR_QP_MODIFY); 155f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.hdr.context = (unsigned long) vq_req; 156f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.rnic_handle = c2dev->adapter_handle; 157f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.qp_handle = qp->adapter_handle; 158f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.ord = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); 159f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.ird = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); 160f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); 161f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); 162f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 163f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (attr_mask & IB_QP_STATE) { 164f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* Ensure the state is valid */ 165ad1f9791e91511fcff21730e55ad8dbc375c79bbKrishna Kumar if (attr->qp_state < 0 || attr->qp_state > IB_QPS_ERR) { 166ad1f9791e91511fcff21730e55ad8dbc375c79bbKrishna Kumar err = -EINVAL; 167ad1f9791e91511fcff21730e55ad8dbc375c79bbKrishna Kumar goto bail0; 168ad1f9791e91511fcff21730e55ad8dbc375c79bbKrishna Kumar } 169f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 170f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.next_qp_state = cpu_to_be32(to_c2_state(attr->qp_state)); 171f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 172f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (attr->qp_state == IB_QPS_ERR) { 173f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker spin_lock_irqsave(&qp->lock, flags); 174f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (qp->cm_id && qp->state == IB_QPS_RTS) { 175f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker pr_debug("Generating CLOSE event for QP-->ERR, " 176f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker "qp=%p, cm_id=%p\n",qp,qp->cm_id); 177f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* Generate an CLOSE event */ 178f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req->cm_id = qp->cm_id; 179f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req->event = IW_CM_EVENT_CLOSE; 180f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 181f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker spin_unlock_irqrestore(&qp->lock, flags); 182f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 183f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker next_state = attr->qp_state; 184f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 185f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } else if (attr_mask & IB_QP_CUR_STATE) { 186f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 187f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (attr->cur_qp_state != IB_QPS_RTR && 188f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker attr->cur_qp_state != IB_QPS_RTS && 189f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker attr->cur_qp_state != IB_QPS_SQD && 190ad1f9791e91511fcff21730e55ad8dbc375c79bbKrishna Kumar attr->cur_qp_state != IB_QPS_SQE) { 191ad1f9791e91511fcff21730e55ad8dbc375c79bbKrishna Kumar err = -EINVAL; 192ad1f9791e91511fcff21730e55ad8dbc375c79bbKrishna Kumar goto bail0; 193ad1f9791e91511fcff21730e55ad8dbc375c79bbKrishna Kumar } else 194f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.next_qp_state = 195f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker cpu_to_be32(to_c2_state(attr->cur_qp_state)); 196f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 197f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker next_state = attr->cur_qp_state; 198f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 199f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } else { 200f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = 0; 201f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker goto bail0; 202f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 203f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 204f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* reference the request struct */ 205f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req_get(c2dev, vq_req); 206f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 207f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = vq_send_wr(c2dev, (union c2wr *) & wr); 208f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (err) { 209f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req_put(c2dev, vq_req); 210f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker goto bail0; 211f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 212f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 213f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = vq_wait_for_reply(c2dev, vq_req); 214f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (err) 215f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker goto bail0; 216f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 217f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker reply = (struct c2wr_qp_modify_rep *) (unsigned long) vq_req->reply_msg; 218f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (!reply) { 219f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = -ENOMEM; 220f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker goto bail0; 221f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 222f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 223f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = c2_errno(reply); 224f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (!err) 225f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker qp->state = next_state; 226f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker#ifdef DEBUG 227f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker else 2283371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison pr_debug("%s: c2_errno=%d\n", __func__, err); 229f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker#endif 230f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 231f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * If we're going to error and generating the event here, then 232f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * we need to remove the reference because there will be no 233f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * close event generated by the adapter 234f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 235f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker spin_lock_irqsave(&qp->lock, flags); 236f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (vq_req->event==IW_CM_EVENT_CLOSE && qp->cm_id) { 237f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker qp->cm_id->rem_ref(qp->cm_id); 238f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker qp->cm_id = NULL; 239f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 240f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker spin_unlock_irqrestore(&qp->lock, flags); 241f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 242f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_repbuf_free(c2dev, reply); 243f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker bail0: 244f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req_free(c2dev, vq_req); 245f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 246f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker pr_debug("%s:%d qp=%p, cur_state=%s\n", 2473371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison __func__, __LINE__, 248f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker qp, 249f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker to_ib_state_str(qp->state)); 250f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return err; 251f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker} 252f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 253f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tuckerint c2_qp_set_read_limits(struct c2_dev *c2dev, struct c2_qp *qp, 254f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker int ord, int ird) 255f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker{ 256f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2wr_qp_modify_req wr; 257f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2wr_qp_modify_rep *reply; 258f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2_vq_req *vq_req; 259f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker int err; 260f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 261f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req = vq_req_alloc(c2dev); 262f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (!vq_req) 263f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return -ENOMEM; 264f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 265f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_wr_set_id(&wr, CCWR_QP_MODIFY); 266f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.hdr.context = (unsigned long) vq_req; 267f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.rnic_handle = c2dev->adapter_handle; 268f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.qp_handle = qp->adapter_handle; 269f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.ord = cpu_to_be32(ord); 270f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.ird = cpu_to_be32(ird); 271f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); 272f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); 273f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.next_qp_state = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); 274f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 275f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* reference the request struct */ 276f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req_get(c2dev, vq_req); 277f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 278f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = vq_send_wr(c2dev, (union c2wr *) & wr); 279f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (err) { 280f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req_put(c2dev, vq_req); 281f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker goto bail0; 282f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 283f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 284f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = vq_wait_for_reply(c2dev, vq_req); 285f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (err) 286f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker goto bail0; 287f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 288f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker reply = (struct c2wr_qp_modify_rep *) (unsigned long) 289f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req->reply_msg; 290f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (!reply) { 291f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = -ENOMEM; 292f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker goto bail0; 293f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 294f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 295f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = c2_errno(reply); 296f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_repbuf_free(c2dev, reply); 297f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker bail0: 298f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req_free(c2dev, vq_req); 299f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return err; 300f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker} 301f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 302f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tuckerstatic int destroy_qp(struct c2_dev *c2dev, struct c2_qp *qp) 303f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker{ 304f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2_vq_req *vq_req; 305f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2wr_qp_destroy_req wr; 306f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2wr_qp_destroy_rep *reply; 307f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker unsigned long flags; 308f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker int err; 309f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 310f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 311f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Allocate a verb request message 312f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 313f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req = vq_req_alloc(c2dev); 314f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (!vq_req) { 315f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return -ENOMEM; 316f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 317f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 318f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 319f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Initialize the WR 320f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 321f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_wr_set_id(&wr, CCWR_QP_DESTROY); 322f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.hdr.context = (unsigned long) vq_req; 323f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.rnic_handle = c2dev->adapter_handle; 324f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.qp_handle = qp->adapter_handle; 325f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 326f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 327f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * reference the request struct. dereferenced in the int handler. 328f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 329f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req_get(c2dev, vq_req); 330f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 331f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker spin_lock_irqsave(&qp->lock, flags); 332f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (qp->cm_id && qp->state == IB_QPS_RTS) { 333f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker pr_debug("destroy_qp: generating CLOSE event for QP-->ERR, " 334f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker "qp=%p, cm_id=%p\n",qp,qp->cm_id); 335f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* Generate an CLOSE event */ 336f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req->qp = qp; 337f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req->cm_id = qp->cm_id; 338f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req->event = IW_CM_EVENT_CLOSE; 339f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 340f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker spin_unlock_irqrestore(&qp->lock, flags); 341f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 342f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 343f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Send WR to adapter 344f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 345f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = vq_send_wr(c2dev, (union c2wr *) & wr); 346f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (err) { 347f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req_put(c2dev, vq_req); 348f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker goto bail0; 349f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 350f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 351f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 352f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Wait for reply from adapter 353f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 354f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = vq_wait_for_reply(c2dev, vq_req); 355f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (err) { 356f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker goto bail0; 357f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 358f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 359f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 360f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Process reply 361f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 362f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker reply = (struct c2wr_qp_destroy_rep *) (unsigned long) (vq_req->reply_msg); 363f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (!reply) { 364f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = -ENOMEM; 365f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker goto bail0; 366f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 367f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 368f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker spin_lock_irqsave(&qp->lock, flags); 369f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (qp->cm_id) { 370f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker qp->cm_id->rem_ref(qp->cm_id); 371f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker qp->cm_id = NULL; 372f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 373f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker spin_unlock_irqrestore(&qp->lock, flags); 374f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 375f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_repbuf_free(c2dev, reply); 376f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker bail0: 377f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req_free(c2dev, vq_req); 378f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return err; 379f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker} 380f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 381f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tuckerstatic int c2_alloc_qpn(struct c2_dev *c2dev, struct c2_qp *qp) 382f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker{ 383f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker int ret; 384f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 385f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker do { 386f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker spin_lock_irq(&c2dev->qp_table.lock); 387f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker ret = idr_get_new_above(&c2dev->qp_table.idr, qp, 388f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2dev->qp_table.last++, &qp->qpn); 389f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker spin_unlock_irq(&c2dev->qp_table.lock); 390f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } while ((ret == -EAGAIN) && 391f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker idr_pre_get(&c2dev->qp_table.idr, GFP_KERNEL)); 392f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return ret; 393f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker} 394f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 395f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tuckerstatic void c2_free_qpn(struct c2_dev *c2dev, int qpn) 396f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker{ 397f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker spin_lock_irq(&c2dev->qp_table.lock); 398f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker idr_remove(&c2dev->qp_table.idr, qpn); 399f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker spin_unlock_irq(&c2dev->qp_table.lock); 400f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker} 401f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 402f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tuckerstruct c2_qp *c2_find_qpn(struct c2_dev *c2dev, int qpn) 403f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker{ 404f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker unsigned long flags; 405f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2_qp *qp; 406f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 407f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker spin_lock_irqsave(&c2dev->qp_table.lock, flags); 408f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker qp = idr_find(&c2dev->qp_table.idr, qpn); 409f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker spin_unlock_irqrestore(&c2dev->qp_table.lock, flags); 410f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return qp; 411f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker} 412f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 413f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tuckerint c2_alloc_qp(struct c2_dev *c2dev, 414f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2_pd *pd, 415f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct ib_qp_init_attr *qp_attrs, struct c2_qp *qp) 416f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker{ 417f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2wr_qp_create_req wr; 418f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2wr_qp_create_rep *reply; 419f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2_vq_req *vq_req; 420f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2_cq *send_cq = to_c2cq(qp_attrs->send_cq); 421f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2_cq *recv_cq = to_c2cq(qp_attrs->recv_cq); 422f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker unsigned long peer_pa; 423f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker u32 q_size, msg_size, mmap_size; 424f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker void __iomem *mmap; 425f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker int err; 426f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 427f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = c2_alloc_qpn(c2dev, qp); 428f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (err) 429f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return err; 430f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker qp->ibqp.qp_num = qp->qpn; 431f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker qp->ibqp.qp_type = IB_QPT_RC; 432f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 433f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* Allocate the SQ and RQ shared pointers */ 434f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker qp->sq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool, 435f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker &qp->sq_mq.shared_dma, GFP_KERNEL); 436f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (!qp->sq_mq.shared) { 437f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = -ENOMEM; 438f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker goto bail0; 439f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 440f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 441f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker qp->rq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool, 442f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker &qp->rq_mq.shared_dma, GFP_KERNEL); 443f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (!qp->rq_mq.shared) { 444f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = -ENOMEM; 445f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker goto bail1; 446f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 447f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 448f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* Allocate the verbs request */ 449f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req = vq_req_alloc(c2dev); 450f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (vq_req == NULL) { 451f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = -ENOMEM; 452f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker goto bail2; 453f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 454f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 455f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* Initialize the work request */ 456f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker memset(&wr, 0, sizeof(wr)); 457f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_wr_set_id(&wr, CCWR_QP_CREATE); 458f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.hdr.context = (unsigned long) vq_req; 459f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.rnic_handle = c2dev->adapter_handle; 460f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.sq_cq_handle = send_cq->adapter_handle; 461f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.rq_cq_handle = recv_cq->adapter_handle; 462f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.sq_depth = cpu_to_be32(qp_attrs->cap.max_send_wr + 1); 463f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.rq_depth = cpu_to_be32(qp_attrs->cap.max_recv_wr + 1); 464f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.srq_handle = 0; 465f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.flags = cpu_to_be32(QP_RDMA_READ | QP_RDMA_WRITE | QP_MW_BIND | 466f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker QP_ZERO_STAG | QP_RDMA_READ_RESPONSE); 467f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.send_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge); 468f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.recv_sgl_depth = cpu_to_be32(qp_attrs->cap.max_recv_sge); 469f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.rdma_write_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge); 470f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.shared_sq_ht = cpu_to_be64(qp->sq_mq.shared_dma); 471f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.shared_rq_ht = cpu_to_be64(qp->rq_mq.shared_dma); 472f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.ord = cpu_to_be32(C2_MAX_ORD_PER_QP); 473f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.ird = cpu_to_be32(C2_MAX_IRD_PER_QP); 474f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.pd_id = pd->pd_id; 475f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.user_context = (unsigned long) qp; 476f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 477f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req_get(c2dev, vq_req); 478f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 479f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* Send the WR to the adapter */ 480f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = vq_send_wr(c2dev, (union c2wr *) & wr); 481f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (err) { 482f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req_put(c2dev, vq_req); 483f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker goto bail3; 484f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 485f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 486f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* Wait for the verb reply */ 487f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = vq_wait_for_reply(c2dev, vq_req); 488f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (err) { 489f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker goto bail3; 490f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 491f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 492f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* Process the reply */ 493f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker reply = (struct c2wr_qp_create_rep *) (unsigned long) (vq_req->reply_msg); 494f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (!reply) { 495f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = -ENOMEM; 496f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker goto bail3; 497f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 498f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 499f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if ((err = c2_wr_get_result(reply)) != 0) { 500f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker goto bail4; 501f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 502f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 503f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* Fill in the kernel QP struct */ 504f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker atomic_set(&qp->refcount, 1); 505f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker qp->adapter_handle = reply->qp_handle; 506f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker qp->state = IB_QPS_RESET; 507f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker qp->send_sgl_depth = qp_attrs->cap.max_send_sge; 508f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker qp->rdma_write_sgl_depth = qp_attrs->cap.max_send_sge; 509f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker qp->recv_sgl_depth = qp_attrs->cap.max_recv_sge; 5104e8e6ee380c3858151165d7455b4954782f145a0Tom Tucker init_waitqueue_head(&qp->wait); 511f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 512f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* Initialize the SQ MQ */ 513f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker q_size = be32_to_cpu(reply->sq_depth); 514f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker msg_size = be32_to_cpu(reply->sq_msg_size); 515f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker peer_pa = c2dev->pa + be32_to_cpu(reply->sq_mq_start); 516f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size); 517f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker mmap = ioremap_nocache(peer_pa, mmap_size); 518f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (!mmap) { 519f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = -ENOMEM; 520f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker goto bail5; 521f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 522f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 523f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_mq_req_init(&qp->sq_mq, 524f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker be32_to_cpu(reply->sq_mq_index), 525f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker q_size, 526f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker msg_size, 527f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker mmap + sizeof(struct c2_mq_shared), /* pool start */ 528f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker mmap, /* peer */ 529f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker C2_MQ_ADAPTER_TARGET); 530f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 531f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* Initialize the RQ mq */ 532f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker q_size = be32_to_cpu(reply->rq_depth); 533f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker msg_size = be32_to_cpu(reply->rq_msg_size); 534f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker peer_pa = c2dev->pa + be32_to_cpu(reply->rq_mq_start); 535f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size); 536f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker mmap = ioremap_nocache(peer_pa, mmap_size); 537f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (!mmap) { 538f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = -ENOMEM; 539f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker goto bail6; 540f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 541f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 542f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_mq_req_init(&qp->rq_mq, 543f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker be32_to_cpu(reply->rq_mq_index), 544f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker q_size, 545f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker msg_size, 546f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker mmap + sizeof(struct c2_mq_shared), /* pool start */ 547f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker mmap, /* peer */ 548f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker C2_MQ_ADAPTER_TARGET); 549f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 550f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_repbuf_free(c2dev, reply); 551f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req_free(c2dev, vq_req); 552f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 553f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return 0; 554f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 555f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker bail6: 556f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker iounmap(qp->sq_mq.peer); 557f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker bail5: 558f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker destroy_qp(c2dev, qp); 559f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker bail4: 560f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_repbuf_free(c2dev, reply); 561f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker bail3: 562f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker vq_req_free(c2dev, vq_req); 563f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker bail2: 564f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_free_mqsp(qp->rq_mq.shared); 565f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker bail1: 566f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_free_mqsp(qp->sq_mq.shared); 567f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker bail0: 568f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_free_qpn(c2dev, qp->qpn); 569f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return err; 570f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker} 571f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 572c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumarstatic inline void c2_lock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq) 573c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar{ 574c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar if (send_cq == recv_cq) 575c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar spin_lock_irq(&send_cq->lock); 576c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar else if (send_cq > recv_cq) { 577c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar spin_lock_irq(&send_cq->lock); 578c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING); 579c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar } else { 580c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar spin_lock_irq(&recv_cq->lock); 581c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); 582c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar } 583c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar} 584c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar 585c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumarstatic inline void c2_unlock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq) 586c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar{ 587c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar if (send_cq == recv_cq) 588c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar spin_unlock_irq(&send_cq->lock); 589c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar else if (send_cq > recv_cq) { 590c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar spin_unlock(&recv_cq->lock); 591c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar spin_unlock_irq(&send_cq->lock); 592c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar } else { 593c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar spin_unlock(&send_cq->lock); 594c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar spin_unlock_irq(&recv_cq->lock); 595c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar } 596c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar} 597c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar 598f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tuckervoid c2_free_qp(struct c2_dev *c2dev, struct c2_qp *qp) 599f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker{ 600f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2_cq *send_cq; 601f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2_cq *recv_cq; 602f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 603f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker send_cq = to_c2cq(qp->ibqp.send_cq); 604f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker recv_cq = to_c2cq(qp->ibqp.recv_cq); 605f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 606f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 607f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Lock CQs here, so that CQ polling code can do QP lookup 608f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * without taking a lock. 609f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 610c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar c2_lock_cqs(send_cq, recv_cq); 611f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_free_qpn(c2dev, qp->qpn); 612c9edea298e52faeb0d4ae875cb712a5d69ba1966Krishna Kumar c2_unlock_cqs(send_cq, recv_cq); 613f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 614f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 61525985edcedea6396277003854657b5f3cb31a628Lucas De Marchi * Destroy qp in the rnic... 616f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 617f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker destroy_qp(c2dev, qp); 618f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 619f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 620f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Mark any unreaped CQEs as null and void. 621f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 622f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_cq_clean(c2dev, qp, send_cq->cqn); 623f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (send_cq != recv_cq) 624f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_cq_clean(c2dev, qp, recv_cq->cqn); 625f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 626f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Unmap the MQs and return the shared pointers 627f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * to the message pool. 628f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 629f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker iounmap(qp->sq_mq.peer); 630f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker iounmap(qp->rq_mq.peer); 631f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_free_mqsp(qp->sq_mq.shared); 632f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_free_mqsp(qp->rq_mq.shared); 633f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 634f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker atomic_dec(&qp->refcount); 635f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wait_event(qp->wait, !atomic_read(&qp->refcount)); 636f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker} 637f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 638f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker/* 639f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Function: move_sgl 640f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * 641f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Description: 642f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Move an SGL from the user's work request struct into a CCIL Work Request 643f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * message, swapping to WR byte order and ensure the total length doesn't 644f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * overflow. 645f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * 646f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * IN: 647f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * dst - ptr to CCIL Work Request message SGL memory. 648f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * src - ptr to the consumers SGL memory. 649f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * 650f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * OUT: none 651f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * 652f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Return: 653f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * CCIL status codes. 654f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 655f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tuckerstatic int 656f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tuckermove_sgl(struct c2_data_addr * dst, struct ib_sge *src, int count, u32 * p_len, 657f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker u8 * actual_count) 658f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker{ 659f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker u32 tot = 0; /* running total */ 660f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker u8 acount = 0; /* running total non-0 len sge's */ 661f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 662f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker while (count > 0) { 663f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 664f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * If the addition of this SGE causes the 665f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * total SGL length to exceed 2^32-1, then 666f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * fail-n-bail. 667f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * 668f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * If the current total plus the next element length 669f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * wraps, then it will go negative and be less than the 670f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * current total... 671f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 672f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if ((tot + src->length) < tot) { 673f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return -EINVAL; 674f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 675f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 676f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Bug: 1456 (as well as 1498 & 1643) 677f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Skip over any sge's supplied with len=0 678f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 679f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (src->length) { 680f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker tot += src->length; 681f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker dst->stag = cpu_to_be32(src->lkey); 682f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker dst->to = cpu_to_be64(src->addr); 683f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker dst->length = cpu_to_be32(src->length); 684f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker dst++; 685f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker acount++; 686f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 687f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker src++; 688f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker count--; 689f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 690f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 691f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (acount == 0) { 692f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 693f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Bug: 1476 (as well as 1498, 1456 and 1643) 694f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Setup the SGL in the WR to make it easier for the RNIC. 695f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * This way, the FW doesn't have to deal with special cases. 696f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Setting length=0 should be sufficient. 697f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 698f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker dst->stag = 0; 699f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker dst->to = 0; 700f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker dst->length = 0; 701f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 702f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 703f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker *p_len = tot; 704f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker *actual_count = acount; 705f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return 0; 706f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker} 707f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 708f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker/* 709f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Function: c2_activity (private function) 710f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * 711f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Description: 712f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Post an mq index to the host->adapter activity fifo. 713f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * 714f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * IN: 715f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * c2dev - ptr to c2dev structure 716f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * mq_index - mq index to post 717f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * shared - value most recently written to shared 718f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * 719f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * OUT: 720f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * 721f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Return: 722f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * none 723f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 724f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tuckerstatic inline void c2_activity(struct c2_dev *c2dev, u32 mq_index, u16 shared) 725f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker{ 726f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 727f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * First read the register to see if the FIFO is full, and if so, 728f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * spin until it's not. This isn't perfect -- there is no 729f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * synchronization among the clients of the register, but in 730f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * practice it prevents multiple CPU from hammering the bus 731f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * with PCI RETRY. Note that when this does happen, the card 732f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * cannot get on the bus and the card and system hang in a 733f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * deadlock -- thus the need for this code. [TOT] 734f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 735e52e6080ca10e0a8ce2a35c86965945cdfa8ed7eTom Tucker while (readl(c2dev->regs + PCI_BAR0_ADAPTER_HINT) & 0x80000000) 736e52e6080ca10e0a8ce2a35c86965945cdfa8ed7eTom Tucker udelay(10); 737f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 738f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker __raw_writel(C2_HINT_MAKE(mq_index, shared), 739f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2dev->regs + PCI_BAR0_ADAPTER_HINT); 740f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker} 741f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 742f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker/* 743f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Function: qp_wr_post 744f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * 745f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Description: 746f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * This in-line function allocates a MQ msg, then moves the host-copy of 747f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * the completed WR into msg. Then it posts the message. 748f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * 749f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * IN: 750f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * q - ptr to user MQ. 751f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * wr - ptr to host-copy of the WR. 752f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * qp - ptr to user qp 753f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * size - Number of bytes to post. Assumed to be divisible by 4. 754f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * 755f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * OUT: none 756f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * 757f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Return: 758f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * CCIL status codes. 759f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 760f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tuckerstatic int qp_wr_post(struct c2_mq *q, union c2wr * wr, struct c2_qp *qp, u32 size) 761f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker{ 762f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker union c2wr *msg; 763f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 764f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker msg = c2_mq_alloc(q); 765f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (msg == NULL) { 766f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return -EINVAL; 767f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 768f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker#ifdef CCMSGMAGIC 769f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker ((c2wr_hdr_t *) wr)->magic = cpu_to_be32(CCWR_MAGIC); 770f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker#endif 771f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 772f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 773f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Since all header fields in the WR are the same as the 774f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * CQE, set the following so the adapter need not. 775f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 776f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_wr_set_result(wr, CCERR_PENDING); 777f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 778f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 779f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Copy the wr down to the adapter 780f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 781f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker memcpy((void *) msg, (void *) wr, size); 782f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 783f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_mq_produce(q); 784f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return 0; 785f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker} 786f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 787f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 788f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tuckerint c2_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr, 789f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct ib_send_wr **bad_wr) 790f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker{ 791f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2_dev *c2dev = to_c2dev(ibqp->device); 792f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2_qp *qp = to_c2qp(ibqp); 793f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker union c2wr wr; 794e52e6080ca10e0a8ce2a35c86965945cdfa8ed7eTom Tucker unsigned long lock_flags; 795f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker int err = 0; 796f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 797f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker u32 flags; 798f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker u32 tot_len; 799f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker u8 actual_sge_count; 800f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker u32 msg_size; 801f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 802c597b0240b686427248b6d4fc8adbe22f9a04c11Frank Zago if (qp->state > IB_QPS_RTS) { 803c597b0240b686427248b6d4fc8adbe22f9a04c11Frank Zago err = -EINVAL; 804c597b0240b686427248b6d4fc8adbe22f9a04c11Frank Zago goto out; 805c597b0240b686427248b6d4fc8adbe22f9a04c11Frank Zago } 806f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 807f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker while (ib_wr) { 808f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 809f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker flags = 0; 810f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.sqwr.sq_hdr.user_hdr.hdr.context = ib_wr->wr_id; 811f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (ib_wr->send_flags & IB_SEND_SIGNALED) { 812f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker flags |= SQ_SIGNALED; 813f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 814f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 815f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker switch (ib_wr->opcode) { 816f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker case IB_WR_SEND: 817139b2db7951ed0c808884cfebca02a884aebe0c3Roland Dreier case IB_WR_SEND_WITH_INV: 818139b2db7951ed0c808884cfebca02a884aebe0c3Roland Dreier if (ib_wr->opcode == IB_WR_SEND) { 819139b2db7951ed0c808884cfebca02a884aebe0c3Roland Dreier if (ib_wr->send_flags & IB_SEND_SOLICITED) 820139b2db7951ed0c808884cfebca02a884aebe0c3Roland Dreier c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE); 821139b2db7951ed0c808884cfebca02a884aebe0c3Roland Dreier else 822139b2db7951ed0c808884cfebca02a884aebe0c3Roland Dreier c2_wr_set_id(&wr, C2_WR_TYPE_SEND); 823139b2db7951ed0c808884cfebca02a884aebe0c3Roland Dreier wr.sqwr.send.remote_stag = 0; 824f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } else { 825139b2db7951ed0c808884cfebca02a884aebe0c3Roland Dreier if (ib_wr->send_flags & IB_SEND_SOLICITED) 826139b2db7951ed0c808884cfebca02a884aebe0c3Roland Dreier c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE_INV); 827139b2db7951ed0c808884cfebca02a884aebe0c3Roland Dreier else 828139b2db7951ed0c808884cfebca02a884aebe0c3Roland Dreier c2_wr_set_id(&wr, C2_WR_TYPE_SEND_INV); 829139b2db7951ed0c808884cfebca02a884aebe0c3Roland Dreier wr.sqwr.send.remote_stag = 830139b2db7951ed0c808884cfebca02a884aebe0c3Roland Dreier cpu_to_be32(ib_wr->ex.invalidate_rkey); 831f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 832f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 833139b2db7951ed0c808884cfebca02a884aebe0c3Roland Dreier msg_size = sizeof(struct c2wr_send_req) + 834139b2db7951ed0c808884cfebca02a884aebe0c3Roland Dreier sizeof(struct c2_data_addr) * ib_wr->num_sge; 835f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (ib_wr->num_sge > qp->send_sgl_depth) { 836f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = -EINVAL; 837f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker break; 838f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 839f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (ib_wr->send_flags & IB_SEND_FENCE) { 840f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker flags |= SQ_READ_FENCE; 841f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 842f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = move_sgl((struct c2_data_addr *) & (wr.sqwr.send.data), 843f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker ib_wr->sg_list, 844f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker ib_wr->num_sge, 845f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker &tot_len, &actual_sge_count); 846f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.sqwr.send.sge_len = cpu_to_be32(tot_len); 847f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_wr_set_sge_count(&wr, actual_sge_count); 848f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker break; 849f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker case IB_WR_RDMA_WRITE: 850f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_WRITE); 851f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker msg_size = sizeof(struct c2wr_rdma_write_req) + 852f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker (sizeof(struct c2_data_addr) * ib_wr->num_sge); 853f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (ib_wr->num_sge > qp->rdma_write_sgl_depth) { 854f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = -EINVAL; 855f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker break; 856f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 857f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (ib_wr->send_flags & IB_SEND_FENCE) { 858f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker flags |= SQ_READ_FENCE; 859f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 860f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.sqwr.rdma_write.remote_stag = 861f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker cpu_to_be32(ib_wr->wr.rdma.rkey); 862f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.sqwr.rdma_write.remote_to = 863f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker cpu_to_be64(ib_wr->wr.rdma.remote_addr); 864f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = move_sgl((struct c2_data_addr *) 865f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker & (wr.sqwr.rdma_write.data), 866f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker ib_wr->sg_list, 867f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker ib_wr->num_sge, 868f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker &tot_len, &actual_sge_count); 869f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.sqwr.rdma_write.sge_len = cpu_to_be32(tot_len); 870f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_wr_set_sge_count(&wr, actual_sge_count); 871f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker break; 872f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker case IB_WR_RDMA_READ: 873f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_READ); 874f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker msg_size = sizeof(struct c2wr_rdma_read_req); 875f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 876f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* IWarp only suppots 1 sge for RDMA reads */ 877f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (ib_wr->num_sge > 1) { 878f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = -EINVAL; 879f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker break; 880f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 881f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 882f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 883f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Move the local and remote stag/to/len into the WR. 884f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 885f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.sqwr.rdma_read.local_stag = 886f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker cpu_to_be32(ib_wr->sg_list->lkey); 887f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.sqwr.rdma_read.local_to = 888f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker cpu_to_be64(ib_wr->sg_list->addr); 889f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.sqwr.rdma_read.remote_stag = 890f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker cpu_to_be32(ib_wr->wr.rdma.rkey); 891f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.sqwr.rdma_read.remote_to = 892f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker cpu_to_be64(ib_wr->wr.rdma.remote_addr); 893f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.sqwr.rdma_read.length = 894f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker cpu_to_be32(ib_wr->sg_list->length); 895f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker break; 896f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker default: 897f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* error */ 898f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker msg_size = 0; 899f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = -EINVAL; 900f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker break; 901f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 902f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 903f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 904f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * If we had an error on the last wr build, then 905f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * break out. Possible errors include bogus WR 906f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * type, and a bogus SGL length... 907f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 908f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (err) { 909f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker break; 910f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 911f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 912f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 913f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Store flags 914f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 915f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_wr_set_flags(&wr, flags); 916f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 917f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 918f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Post the puppy! 919f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 920e52e6080ca10e0a8ce2a35c86965945cdfa8ed7eTom Tucker spin_lock_irqsave(&qp->lock, lock_flags); 921f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = qp_wr_post(&qp->sq_mq, &wr, qp, msg_size); 922f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (err) { 923e52e6080ca10e0a8ce2a35c86965945cdfa8ed7eTom Tucker spin_unlock_irqrestore(&qp->lock, lock_flags); 924f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker break; 925f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 926f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 927f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 928f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Enqueue mq index to activity FIFO. 929f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 930f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_activity(c2dev, qp->sq_mq.index, qp->sq_mq.hint_count); 931e52e6080ca10e0a8ce2a35c86965945cdfa8ed7eTom Tucker spin_unlock_irqrestore(&qp->lock, lock_flags); 932f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 933f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker ib_wr = ib_wr->next; 934f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 935f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 936c597b0240b686427248b6d4fc8adbe22f9a04c11Frank Zagoout: 937f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (err) 938f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker *bad_wr = ib_wr; 939f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return err; 940f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker} 941f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 942f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tuckerint c2_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr, 943f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct ib_recv_wr **bad_wr) 944f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker{ 945f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2_dev *c2dev = to_c2dev(ibqp->device); 946f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker struct c2_qp *qp = to_c2qp(ibqp); 947f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker union c2wr wr; 948e52e6080ca10e0a8ce2a35c86965945cdfa8ed7eTom Tucker unsigned long lock_flags; 949f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker int err = 0; 950f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 951c597b0240b686427248b6d4fc8adbe22f9a04c11Frank Zago if (qp->state > IB_QPS_RTS) { 952c597b0240b686427248b6d4fc8adbe22f9a04c11Frank Zago err = -EINVAL; 953c597b0240b686427248b6d4fc8adbe22f9a04c11Frank Zago goto out; 954c597b0240b686427248b6d4fc8adbe22f9a04c11Frank Zago } 955f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 956f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 957f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Try and post each work request 958f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 959f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker while (ib_wr) { 960f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker u32 tot_len; 961f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker u8 actual_sge_count; 962f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 963f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (ib_wr->num_sge > qp->recv_sgl_depth) { 964f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = -EINVAL; 965f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker break; 966f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 967f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 968f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 969f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Create local host-copy of the WR 970f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 971f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker wr.rqwr.rq_hdr.user_hdr.hdr.context = ib_wr->wr_id; 972f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_wr_set_id(&wr, CCWR_RECV); 973f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_wr_set_flags(&wr, 0); 974f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 975f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* sge_count is limited to eight bits. */ 976f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker BUG_ON(ib_wr->num_sge >= 256); 977f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = move_sgl((struct c2_data_addr *) & (wr.rqwr.data), 978f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker ib_wr->sg_list, 979f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker ib_wr->num_sge, &tot_len, &actual_sge_count); 980f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_wr_set_sge_count(&wr, actual_sge_count); 981f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 982f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 983f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * If we had an error on the last wr build, then 984f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * break out. Possible errors include bogus WR 985f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * type, and a bogus SGL length... 986f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 987f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (err) { 988f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker break; 989f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 990f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 991e52e6080ca10e0a8ce2a35c86965945cdfa8ed7eTom Tucker spin_lock_irqsave(&qp->lock, lock_flags); 992f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker err = qp_wr_post(&qp->rq_mq, &wr, qp, qp->rq_mq.msg_size); 993f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (err) { 994e52e6080ca10e0a8ce2a35c86965945cdfa8ed7eTom Tucker spin_unlock_irqrestore(&qp->lock, lock_flags); 995f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker break; 996f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 997f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 998f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker /* 999f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker * Enqueue mq index to activity FIFO 1000f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker */ 1001f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker c2_activity(c2dev, qp->rq_mq.index, qp->rq_mq.hint_count); 1002e52e6080ca10e0a8ce2a35c86965945cdfa8ed7eTom Tucker spin_unlock_irqrestore(&qp->lock, lock_flags); 1003f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 1004f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker ib_wr = ib_wr->next; 1005f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker } 1006f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 1007c597b0240b686427248b6d4fc8adbe22f9a04c11Frank Zagoout: 1008f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker if (err) 1009f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker *bad_wr = ib_wr; 1010f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker return err; 1011f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker} 1012f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 1013f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tuckervoid __devinit c2_init_qp_table(struct c2_dev *c2dev) 1014f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker{ 1015f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker spin_lock_init(&c2dev->qp_table.lock); 1016f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker idr_init(&c2dev->qp_table.idr); 1017f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker} 1018f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker 1019f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tuckervoid __devexit c2_cleanup_qp_table(struct c2_dev *c2dev) 1020f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker{ 1021f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker idr_destroy(&c2dev->qp_table.idr); 1022f94b533d091a42da92d908eb7b3f9ade1923f90dTom Tucker} 1023