1b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise/* 2b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * Copyright (c) 2006 Chelsio, Inc. All rights reserved. 3b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * 4b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * This software is available to you under a choice of one of two 5b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * licenses. You may choose to be licensed under the terms of the GNU 6b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * General Public License (GPL) Version 2, available from the file 7b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * COPYING in the main directory of this source tree, or the 8b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * OpenIB.org BSD license below: 9b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * 10b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * Redistribution and use in source and binary forms, with or 11b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * without modification, are permitted provided that the following 12b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * conditions are met: 13b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * 14b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * - Redistributions of source code must retain the above 15b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * copyright notice, this list of conditions and the following 16b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * disclaimer. 17b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * 18b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * - Redistributions in binary form must reproduce the above 19b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * copyright notice, this list of conditions and the following 20b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * disclaimer in the documentation and/or other materials 21b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * provided with the distribution. 22b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * 23b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * SOFTWARE. 31b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise */ 32d43c36dc6b357fa1806800f18aa30123c747a6d1Alexey Dobriyan#include <linux/sched.h> 335a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/gfp.h> 34b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#include "iwch_provider.h" 35b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#include "iwch.h" 36b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#include "iwch_cm.h" 37b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#include "cxio_hal.h" 384ab928f69208d240d3681336f34589e4b151824fSteve Wise#include "cxio_resource.h" 39b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 40b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#define NO_SUPPORT -1 41b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 424ab928f69208d240d3681336f34589e4b151824fSteve Wisestatic int build_rdma_send(union t3_wr *wqe, struct ib_send_wr *wr, 432b540355cd2f46c5445030995e72c4b4fb2b775eAdrian Bunk u8 * flit_cnt) 44b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 45b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int i; 46b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 plen; 47b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 48b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise switch (wr->opcode) { 49b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IB_WR_SEND: 50b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (wr->send_flags & IB_SEND_SOLICITED) 51b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.rdmaop = T3_SEND_WITH_SE; 52b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise else 53b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.rdmaop = T3_SEND; 54b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.rem_stag = 0; 55b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 56e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise case IB_WR_SEND_WITH_INV: 57e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise if (wr->send_flags & IB_SEND_SOLICITED) 58e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise wqe->send.rdmaop = T3_SEND_WITH_SE_INV; 59e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise else 60e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise wqe->send.rdmaop = T3_SEND_WITH_INV; 61e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise wqe->send.rem_stag = cpu_to_be32(wr->ex.invalidate_rkey); 62b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 63b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise default: 64e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise return -EINVAL; 65b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 66b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (wr->num_sge > T3_MAX_SGE) 67b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 68b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.reserved[0] = 0; 69b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.reserved[1] = 0; 70b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.reserved[2] = 0; 71e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise plen = 0; 72e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise for (i = 0; i < wr->num_sge; i++) { 73e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise if ((plen + wr->sg_list[i].length) < plen) 74e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise return -EMSGSIZE; 75e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise 76e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise plen += wr->sg_list[i].length; 77e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise wqe->send.sgl[i].stag = cpu_to_be32(wr->sg_list[i].lkey); 78e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise wqe->send.sgl[i].len = cpu_to_be32(wr->sg_list[i].length); 79e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise wqe->send.sgl[i].to = cpu_to_be64(wr->sg_list[i].addr); 80b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 81e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise wqe->send.num_sgle = cpu_to_be32(wr->num_sge); 82e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise *flit_cnt = 4 + ((wr->num_sge) << 1); 83b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.plen = cpu_to_be32(plen); 84b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return 0; 85b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 86b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 874ab928f69208d240d3681336f34589e4b151824fSteve Wisestatic int build_rdma_write(union t3_wr *wqe, struct ib_send_wr *wr, 882b540355cd2f46c5445030995e72c4b4fb2b775eAdrian Bunk u8 *flit_cnt) 89b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 90b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int i; 91b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 plen; 92b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (wr->num_sge > T3_MAX_SGE) 93b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 94b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.rdmaop = T3_RDMA_WRITE; 95b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.reserved[0] = 0; 96b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.reserved[1] = 0; 97b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.reserved[2] = 0; 98b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.stag_sink = cpu_to_be32(wr->wr.rdma.rkey); 99b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.to_sink = cpu_to_be64(wr->wr.rdma.remote_addr); 100b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 101b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) { 102b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise plen = 4; 1030f39cf3d54e67a705773fd0ec56ca3dcd3e9272fRoland Dreier wqe->write.sgl[0].stag = wr->ex.imm_data; 1049c3da0991754d480328eeaa2b90cb231a1cea9b6Harvey Harrison wqe->write.sgl[0].len = cpu_to_be32(0); 1059c3da0991754d480328eeaa2b90cb231a1cea9b6Harvey Harrison wqe->write.num_sgle = cpu_to_be32(0); 106b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *flit_cnt = 6; 107b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } else { 108b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise plen = 0; 109b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise for (i = 0; i < wr->num_sge; i++) { 110b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if ((plen + wr->sg_list[i].length) < plen) { 111b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EMSGSIZE; 112b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 113b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise plen += wr->sg_list[i].length; 114b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.sgl[i].stag = 115b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise cpu_to_be32(wr->sg_list[i].lkey); 116b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.sgl[i].len = 117b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise cpu_to_be32(wr->sg_list[i].length); 118b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.sgl[i].to = 119b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise cpu_to_be64(wr->sg_list[i].addr); 120b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 121b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.num_sgle = cpu_to_be32(wr->num_sge); 122b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *flit_cnt = 5 + ((wr->num_sge) << 1); 123b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 124b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.plen = cpu_to_be32(plen); 125b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return 0; 126b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 127b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 1284ab928f69208d240d3681336f34589e4b151824fSteve Wisestatic int build_rdma_read(union t3_wr *wqe, struct ib_send_wr *wr, 1292b540355cd2f46c5445030995e72c4b4fb2b775eAdrian Bunk u8 *flit_cnt) 130b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 131b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (wr->num_sge > 1) 132b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 133b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->read.rdmaop = T3_READ_REQ; 134e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise if (wr->opcode == IB_WR_RDMA_READ_WITH_INV) 135e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise wqe->read.local_inv = 1; 136e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise else 137e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise wqe->read.local_inv = 0; 138b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->read.reserved[0] = 0; 139b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->read.reserved[1] = 0; 140b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->read.rem_stag = cpu_to_be32(wr->wr.rdma.rkey); 141b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->read.rem_to = cpu_to_be64(wr->wr.rdma.remote_addr); 142b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->read.local_stag = cpu_to_be32(wr->sg_list[0].lkey); 143b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->read.local_len = cpu_to_be32(wr->sg_list[0].length); 144b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->read.local_to = cpu_to_be64(wr->sg_list[0].addr); 145b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *flit_cnt = sizeof(struct t3_rdma_read_wr) >> 3; 146b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return 0; 147b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 148b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 1494ab928f69208d240d3681336f34589e4b151824fSteve Wisestatic int build_fastreg(union t3_wr *wqe, struct ib_send_wr *wr, 150e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise u8 *flit_cnt, int *wr_cnt, struct t3_wq *wq) 151e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise{ 152e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise int i; 153e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise __be64 *p; 154e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise 155e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise if (wr->wr.fast_reg.page_list_len > T3_MAX_FASTREG_DEPTH) 156e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise return -EINVAL; 157e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise *wr_cnt = 1; 158e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise wqe->fastreg.stag = cpu_to_be32(wr->wr.fast_reg.rkey); 159e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise wqe->fastreg.len = cpu_to_be32(wr->wr.fast_reg.length); 160e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise wqe->fastreg.va_base_hi = cpu_to_be32(wr->wr.fast_reg.iova_start >> 32); 161e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise wqe->fastreg.va_base_lo_fbo = 162e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise cpu_to_be32(wr->wr.fast_reg.iova_start & 0xffffffff); 163e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise wqe->fastreg.page_type_perms = cpu_to_be32( 164e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise V_FR_PAGE_COUNT(wr->wr.fast_reg.page_list_len) | 165e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise V_FR_PAGE_SIZE(wr->wr.fast_reg.page_shift-12) | 166e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise V_FR_TYPE(TPT_VATO) | 167e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise V_FR_PERMS(iwch_ib_to_tpt_access(wr->wr.fast_reg.access_flags))); 168e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise p = &wqe->fastreg.pbl_addrs[0]; 169e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise for (i = 0; i < wr->wr.fast_reg.page_list_len; i++, p++) { 170e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise 171e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise /* If we need a 2nd WR, then set it up */ 172e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise if (i == T3_MAX_FASTREG_FRAG) { 173e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise *wr_cnt = 2; 174e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise wqe = (union t3_wr *)(wq->queue + 175e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise Q_PTR2IDX((wq->wptr+1), wq->size_log2)); 176e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise build_fw_riwrh((void *)wqe, T3_WR_FASTREG, 0, 177e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise Q_GENBIT(wq->wptr + 1, wq->size_log2), 178e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise 0, 1 + wr->wr.fast_reg.page_list_len - T3_MAX_FASTREG_FRAG, 179e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise T3_EOP); 180e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise 181e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise p = &wqe->pbl_frag.pbl_addrs[0]; 182e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise } 183e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise *p = cpu_to_be64((u64)wr->wr.fast_reg.page_list->page_list[i]); 184e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise } 185e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise *flit_cnt = 5 + wr->wr.fast_reg.page_list_len; 186e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise if (*flit_cnt > 15) 187e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise *flit_cnt = 15; 188e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise return 0; 189e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise} 190e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise 1914ab928f69208d240d3681336f34589e4b151824fSteve Wisestatic int build_inv_stag(union t3_wr *wqe, struct ib_send_wr *wr, 192e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise u8 *flit_cnt) 193e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise{ 194e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise wqe->local_inv.stag = cpu_to_be32(wr->ex.invalidate_rkey); 195e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise wqe->local_inv.reserved = 0; 196e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise *flit_cnt = sizeof(struct t3_local_inv_wr) >> 3; 197e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise return 0; 198e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise} 199e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise 2002b540355cd2f46c5445030995e72c4b4fb2b775eAdrian Bunkstatic int iwch_sgl2pbl_map(struct iwch_dev *rhp, struct ib_sge *sg_list, 2012b540355cd2f46c5445030995e72c4b4fb2b775eAdrian Bunk u32 num_sgle, u32 * pbl_addr, u8 * page_size) 202b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 203b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int i; 204b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_mr *mhp; 205900f4c16c338f742b80f3aa500e12ceb017e86afSteve Wise u64 offset; 206b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise for (i = 0; i < num_sgle; i++) { 207b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 208b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp = get_mhp(rhp, (sg_list[i].lkey) >> 8); 209b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!mhp) { 2103371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison PDBG("%s %d\n", __func__, __LINE__); 211b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EIO; 212b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 213b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!mhp->attr.state) { 2143371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison PDBG("%s %d\n", __func__, __LINE__); 215b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EIO; 216b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 217b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (mhp->attr.zbva) { 2183371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison PDBG("%s %d\n", __func__, __LINE__); 219b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EIO; 220b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 221b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 222b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (sg_list[i].addr < mhp->attr.va_fbo) { 2233371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison PDBG("%s %d\n", __func__, __LINE__); 224b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 225b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 226b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (sg_list[i].addr + ((u64) sg_list[i].length) < 227b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sg_list[i].addr) { 2283371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison PDBG("%s %d\n", __func__, __LINE__); 229b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 230b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 231b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (sg_list[i].addr + ((u64) sg_list[i].length) > 232b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp->attr.va_fbo + ((u64) mhp->attr.len)) { 2333371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison PDBG("%s %d\n", __func__, __LINE__); 234b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 235b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 236b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise offset = sg_list[i].addr - mhp->attr.va_fbo; 237900f4c16c338f742b80f3aa500e12ceb017e86afSteve Wise offset += mhp->attr.va_fbo & 238900f4c16c338f742b80f3aa500e12ceb017e86afSteve Wise ((1UL << (12 + mhp->attr.page_size)) - 1); 239b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise pbl_addr[i] = ((mhp->attr.pbl_addr - 240b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise rhp->rdev.rnic_info.pbl_base) >> 3) + 241b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (offset >> (12 + mhp->attr.page_size)); 242b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise page_size[i] = mhp->attr.page_size; 243b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 244b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return 0; 245b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 246b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 2474ab928f69208d240d3681336f34589e4b151824fSteve Wisestatic int build_rdma_recv(struct iwch_qp *qhp, union t3_wr *wqe, 2482b540355cd2f46c5445030995e72c4b4fb2b775eAdrian Bunk struct ib_recv_wr *wr) 249b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 2504ab928f69208d240d3681336f34589e4b151824fSteve Wise int i, err = 0; 2514ab928f69208d240d3681336f34589e4b151824fSteve Wise u32 pbl_addr[T3_MAX_SGE]; 2524ab928f69208d240d3681336f34589e4b151824fSteve Wise u8 page_size[T3_MAX_SGE]; 2534ab928f69208d240d3681336f34589e4b151824fSteve Wise 2544ab928f69208d240d3681336f34589e4b151824fSteve Wise err = iwch_sgl2pbl_map(qhp->rhp, wr->sg_list, wr->num_sge, pbl_addr, 2554ab928f69208d240d3681336f34589e4b151824fSteve Wise page_size); 2564ab928f69208d240d3681336f34589e4b151824fSteve Wise if (err) 2574ab928f69208d240d3681336f34589e4b151824fSteve Wise return err; 2584ab928f69208d240d3681336f34589e4b151824fSteve Wise wqe->recv.pagesz[0] = page_size[0]; 2594ab928f69208d240d3681336f34589e4b151824fSteve Wise wqe->recv.pagesz[1] = page_size[1]; 2604ab928f69208d240d3681336f34589e4b151824fSteve Wise wqe->recv.pagesz[2] = page_size[2]; 2614ab928f69208d240d3681336f34589e4b151824fSteve Wise wqe->recv.pagesz[3] = page_size[3]; 262b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->recv.num_sgle = cpu_to_be32(wr->num_sge); 263b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise for (i = 0; i < wr->num_sge; i++) { 264b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->recv.sgl[i].stag = cpu_to_be32(wr->sg_list[i].lkey); 265b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->recv.sgl[i].len = cpu_to_be32(wr->sg_list[i].length); 2664ab928f69208d240d3681336f34589e4b151824fSteve Wise 2674ab928f69208d240d3681336f34589e4b151824fSteve Wise /* to in the WQE == the offset into the page */ 268426328963078f644c7194403a308588cf684d4c6Steve Wise wqe->recv.sgl[i].to = cpu_to_be64(((u32)wr->sg_list[i].addr) & 269426328963078f644c7194403a308588cf684d4c6Steve Wise ((1UL << (12 + page_size[i])) - 1)); 2704ab928f69208d240d3681336f34589e4b151824fSteve Wise 2714ab928f69208d240d3681336f34589e4b151824fSteve Wise /* pbl_addr is the adapters address in the PBL */ 2724ab928f69208d240d3681336f34589e4b151824fSteve Wise wqe->recv.pbl_addr[i] = cpu_to_be32(pbl_addr[i]); 2734ab928f69208d240d3681336f34589e4b151824fSteve Wise } 2744ab928f69208d240d3681336f34589e4b151824fSteve Wise for (; i < T3_MAX_SGE; i++) { 2754ab928f69208d240d3681336f34589e4b151824fSteve Wise wqe->recv.sgl[i].stag = 0; 2764ab928f69208d240d3681336f34589e4b151824fSteve Wise wqe->recv.sgl[i].len = 0; 2774ab928f69208d240d3681336f34589e4b151824fSteve Wise wqe->recv.sgl[i].to = 0; 2784ab928f69208d240d3681336f34589e4b151824fSteve Wise wqe->recv.pbl_addr[i] = 0; 2794ab928f69208d240d3681336f34589e4b151824fSteve Wise } 2804ab928f69208d240d3681336f34589e4b151824fSteve Wise qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr, 2814ab928f69208d240d3681336f34589e4b151824fSteve Wise qhp->wq.rq_size_log2)].wr_id = wr->wr_id; 2824ab928f69208d240d3681336f34589e4b151824fSteve Wise qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr, 2834ab928f69208d240d3681336f34589e4b151824fSteve Wise qhp->wq.rq_size_log2)].pbl_addr = 0; 2844ab928f69208d240d3681336f34589e4b151824fSteve Wise return 0; 2854ab928f69208d240d3681336f34589e4b151824fSteve Wise} 2864ab928f69208d240d3681336f34589e4b151824fSteve Wise 2874ab928f69208d240d3681336f34589e4b151824fSteve Wisestatic int build_zero_stag_recv(struct iwch_qp *qhp, union t3_wr *wqe, 2884ab928f69208d240d3681336f34589e4b151824fSteve Wise struct ib_recv_wr *wr) 2894ab928f69208d240d3681336f34589e4b151824fSteve Wise{ 2904ab928f69208d240d3681336f34589e4b151824fSteve Wise int i; 2914ab928f69208d240d3681336f34589e4b151824fSteve Wise u32 pbl_addr; 2924ab928f69208d240d3681336f34589e4b151824fSteve Wise u32 pbl_offset; 2934ab928f69208d240d3681336f34589e4b151824fSteve Wise 2944ab928f69208d240d3681336f34589e4b151824fSteve Wise 2954ab928f69208d240d3681336f34589e4b151824fSteve Wise /* 2964ab928f69208d240d3681336f34589e4b151824fSteve Wise * The T3 HW requires the PBL in the HW recv descriptor to reference 2974ab928f69208d240d3681336f34589e4b151824fSteve Wise * a PBL entry. So we allocate the max needed PBL memory here and pass 2984ab928f69208d240d3681336f34589e4b151824fSteve Wise * it to the uP in the recv WR. The uP will build the PBL and setup 2994ab928f69208d240d3681336f34589e4b151824fSteve Wise * the HW recv descriptor. 3004ab928f69208d240d3681336f34589e4b151824fSteve Wise */ 3014ab928f69208d240d3681336f34589e4b151824fSteve Wise pbl_addr = cxio_hal_pblpool_alloc(&qhp->rhp->rdev, T3_STAG0_PBL_SIZE); 3024ab928f69208d240d3681336f34589e4b151824fSteve Wise if (!pbl_addr) 3034ab928f69208d240d3681336f34589e4b151824fSteve Wise return -ENOMEM; 3044ab928f69208d240d3681336f34589e4b151824fSteve Wise 3054ab928f69208d240d3681336f34589e4b151824fSteve Wise /* 3064ab928f69208d240d3681336f34589e4b151824fSteve Wise * Compute the 8B aligned offset. 3074ab928f69208d240d3681336f34589e4b151824fSteve Wise */ 3084ab928f69208d240d3681336f34589e4b151824fSteve Wise pbl_offset = (pbl_addr - qhp->rhp->rdev.rnic_info.pbl_base) >> 3; 3094ab928f69208d240d3681336f34589e4b151824fSteve Wise 3104ab928f69208d240d3681336f34589e4b151824fSteve Wise wqe->recv.num_sgle = cpu_to_be32(wr->num_sge); 3114ab928f69208d240d3681336f34589e4b151824fSteve Wise 3124ab928f69208d240d3681336f34589e4b151824fSteve Wise for (i = 0; i < wr->num_sge; i++) { 3134ab928f69208d240d3681336f34589e4b151824fSteve Wise 3144ab928f69208d240d3681336f34589e4b151824fSteve Wise /* 3154ab928f69208d240d3681336f34589e4b151824fSteve Wise * Use a 128MB page size. This and an imposed 128MB 3164ab928f69208d240d3681336f34589e4b151824fSteve Wise * sge length limit allows us to require only a 2-entry HW 3174ab928f69208d240d3681336f34589e4b151824fSteve Wise * PBL for each SGE. This restriction is acceptable since 3184ab928f69208d240d3681336f34589e4b151824fSteve Wise * since it is not possible to allocate 128MB of contiguous 3194ab928f69208d240d3681336f34589e4b151824fSteve Wise * DMA coherent memory! 3204ab928f69208d240d3681336f34589e4b151824fSteve Wise */ 3214ab928f69208d240d3681336f34589e4b151824fSteve Wise if (wr->sg_list[i].length > T3_STAG0_MAX_PBE_LEN) 3224ab928f69208d240d3681336f34589e4b151824fSteve Wise return -EINVAL; 3234ab928f69208d240d3681336f34589e4b151824fSteve Wise wqe->recv.pagesz[i] = T3_STAG0_PAGE_SHIFT; 3244ab928f69208d240d3681336f34589e4b151824fSteve Wise 3254ab928f69208d240d3681336f34589e4b151824fSteve Wise /* 3264ab928f69208d240d3681336f34589e4b151824fSteve Wise * T3 restricts a recv to all zero-stag or all non-zero-stag. 3274ab928f69208d240d3681336f34589e4b151824fSteve Wise */ 3284ab928f69208d240d3681336f34589e4b151824fSteve Wise if (wr->sg_list[i].lkey != 0) 3294ab928f69208d240d3681336f34589e4b151824fSteve Wise return -EINVAL; 3304ab928f69208d240d3681336f34589e4b151824fSteve Wise wqe->recv.sgl[i].stag = 0; 3314ab928f69208d240d3681336f34589e4b151824fSteve Wise wqe->recv.sgl[i].len = cpu_to_be32(wr->sg_list[i].length); 332457fe7b8a6822907cbe65897dc81b83d9df5bcbfSteve Wise wqe->recv.sgl[i].to = cpu_to_be64(wr->sg_list[i].addr); 3334ab928f69208d240d3681336f34589e4b151824fSteve Wise wqe->recv.pbl_addr[i] = cpu_to_be32(pbl_offset); 3344ab928f69208d240d3681336f34589e4b151824fSteve Wise pbl_offset += 2; 335b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 336b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise for (; i < T3_MAX_SGE; i++) { 3374ab928f69208d240d3681336f34589e4b151824fSteve Wise wqe->recv.pagesz[i] = 0; 338b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->recv.sgl[i].stag = 0; 339b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->recv.sgl[i].len = 0; 340b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->recv.sgl[i].to = 0; 3414ab928f69208d240d3681336f34589e4b151824fSteve Wise wqe->recv.pbl_addr[i] = 0; 342b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 3434ab928f69208d240d3681336f34589e4b151824fSteve Wise qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr, 3444ab928f69208d240d3681336f34589e4b151824fSteve Wise qhp->wq.rq_size_log2)].wr_id = wr->wr_id; 3454ab928f69208d240d3681336f34589e4b151824fSteve Wise qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr, 3464ab928f69208d240d3681336f34589e4b151824fSteve Wise qhp->wq.rq_size_log2)].pbl_addr = pbl_addr; 347b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return 0; 348b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 349b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 350b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wiseint iwch_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 351b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct ib_send_wr **bad_wr) 352b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 353b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int err = 0; 35421609ae3efa42f4118ce741f7e55d66d716cb17cRoland Dreier u8 uninitialized_var(t3_wr_flit_cnt); 355b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise enum t3_wr_opcode t3_wr_opcode = 0; 356b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise enum t3_wr_flags t3_wr_flags; 357b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_qp *qhp; 358b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 idx; 359b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise union t3_wr *wqe; 360b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 num_wrs; 361b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise unsigned long flag; 362b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct t3_swsq *sqp; 363e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise int wr_cnt = 1; 364b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 365b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp = to_iwch_qp(ibqp); 366b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_lock_irqsave(&qhp->lock, flag); 367b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (qhp->attr.state > IWCH_QP_STATE_RTS) { 368b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 36948617f862f9e58ca2a609fea6a76733aff55d672Frank Zago err = -EINVAL; 37048617f862f9e58ca2a609fea6a76733aff55d672Frank Zago goto out; 371b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 372b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, 373b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->wq.sq_size_log2); 3743d4f9a28e0f543e2a633d54f0f37f6e81a7701cdDan Carpenter if (num_wrs == 0) { 375b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 37648617f862f9e58ca2a609fea6a76733aff55d672Frank Zago err = -ENOMEM; 37748617f862f9e58ca2a609fea6a76733aff55d672Frank Zago goto out; 378b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 379b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise while (wr) { 380b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (num_wrs == 0) { 381b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise err = -ENOMEM; 382b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 383b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 384b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); 385b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe = (union t3_wr *) (qhp->wq.queue + idx); 386b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise t3_wr_flags = 0; 387b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (wr->send_flags & IB_SEND_SOLICITED) 388b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise t3_wr_flags |= T3_SOLICITED_EVENT_FLAG; 389b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (wr->send_flags & IB_SEND_SIGNALED) 390b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise t3_wr_flags |= T3_COMPLETION_FLAG; 391b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp = qhp->wq.sq + 392b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2); 393b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise switch (wr->opcode) { 394b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IB_WR_SEND: 395e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise case IB_WR_SEND_WITH_INV: 396e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise if (wr->send_flags & IB_SEND_FENCE) 397e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise t3_wr_flags |= T3_READ_FENCE_FLAG; 398b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise t3_wr_opcode = T3_WR_SEND; 3994ab928f69208d240d3681336f34589e4b151824fSteve Wise err = build_rdma_send(wqe, wr, &t3_wr_flit_cnt); 400b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 401b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IB_WR_RDMA_WRITE: 402b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IB_WR_RDMA_WRITE_WITH_IMM: 403b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise t3_wr_opcode = T3_WR_WRITE; 4044ab928f69208d240d3681336f34589e4b151824fSteve Wise err = build_rdma_write(wqe, wr, &t3_wr_flit_cnt); 405b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 406b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IB_WR_RDMA_READ: 407e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise case IB_WR_RDMA_READ_WITH_INV: 408b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise t3_wr_opcode = T3_WR_READ; 409b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise t3_wr_flags = 0; /* T3 reads are always signaled */ 4104ab928f69208d240d3681336f34589e4b151824fSteve Wise err = build_rdma_read(wqe, wr, &t3_wr_flit_cnt); 411b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (err) 412b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 413b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->read_len = wqe->read.local_len; 414b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!qhp->wq.oldest_read) 415b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->wq.oldest_read = sqp; 416b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 417e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise case IB_WR_FAST_REG_MR: 418e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise t3_wr_opcode = T3_WR_FASTREG; 4194ab928f69208d240d3681336f34589e4b151824fSteve Wise err = build_fastreg(wqe, wr, &t3_wr_flit_cnt, 420e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise &wr_cnt, &qhp->wq); 421e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise break; 422e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise case IB_WR_LOCAL_INV: 423e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise if (wr->send_flags & IB_SEND_FENCE) 424e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise t3_wr_flags |= T3_LOCAL_FENCE_FLAG; 425e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise t3_wr_opcode = T3_WR_INV_STAG; 4264ab928f69208d240d3681336f34589e4b151824fSteve Wise err = build_inv_stag(wqe, wr, &t3_wr_flit_cnt); 427e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise break; 428b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise default: 4293371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison PDBG("%s post of type=%d TBD!\n", __func__, 430b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wr->opcode); 431b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise err = -EINVAL; 432b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 43348617f862f9e58ca2a609fea6a76733aff55d672Frank Zago if (err) 434b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 435b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.wrid.id0.hi = qhp->wq.sq_wptr; 436b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->wr_id = wr->wr_id; 437b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->opcode = wr2opcode(t3_wr_opcode); 438b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->sq_wptr = qhp->wq.sq_wptr; 439b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->complete = 0; 440b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->signaled = (wr->send_flags & IB_SEND_SIGNALED); 441b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 442b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise build_fw_riwrh((void *) wqe, t3_wr_opcode, t3_wr_flags, 443b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 444e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise 0, t3_wr_flit_cnt, 445e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise (wr_cnt == 1) ? T3_SOPEOP : T3_SOP); 446b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s cookie 0x%llx wq idx 0x%x swsq idx %ld opcode %d\n", 4473371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison __func__, (unsigned long long) wr->wr_id, idx, 448b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2), 449b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->opcode); 450b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wr = wr->next; 451b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise num_wrs--; 452e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise qhp->wq.wptr += wr_cnt; 453b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ++(qhp->wq.sq_wptr); 454b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 455b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 456e998f245c4b2d36ae2c35446e54ccbf1fb29d9deSteve Wise if (cxio_wq_db_enabled(&qhp->wq)) 457e998f245c4b2d36ae2c35446e54ccbf1fb29d9deSteve Wise ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); 45848617f862f9e58ca2a609fea6a76733aff55d672Frank Zago 45948617f862f9e58ca2a609fea6a76733aff55d672Frank Zagoout: 46048617f862f9e58ca2a609fea6a76733aff55d672Frank Zago if (err) 46148617f862f9e58ca2a609fea6a76733aff55d672Frank Zago *bad_wr = wr; 462b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return err; 463b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 464b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 465b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wiseint iwch_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, 466b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct ib_recv_wr **bad_wr) 467b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 468b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int err = 0; 469b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_qp *qhp; 470b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 idx; 471b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise union t3_wr *wqe; 472b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 num_wrs; 473b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise unsigned long flag; 474b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 475b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp = to_iwch_qp(ibqp); 476b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_lock_irqsave(&qhp->lock, flag); 477b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (qhp->attr.state > IWCH_QP_STATE_RTS) { 478b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 47948617f862f9e58ca2a609fea6a76733aff55d672Frank Zago err = -EINVAL; 48048617f862f9e58ca2a609fea6a76733aff55d672Frank Zago goto out; 481b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 482b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise num_wrs = Q_FREECNT(qhp->wq.rq_rptr, qhp->wq.rq_wptr, 483b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->wq.rq_size_log2) - 1; 484b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!wr) { 485b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 48648617f862f9e58ca2a609fea6a76733aff55d672Frank Zago err = -ENOMEM; 48748617f862f9e58ca2a609fea6a76733aff55d672Frank Zago goto out; 488b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 489b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise while (wr) { 4904ab928f69208d240d3681336f34589e4b151824fSteve Wise if (wr->num_sge > T3_MAX_SGE) { 4914ab928f69208d240d3681336f34589e4b151824fSteve Wise err = -EINVAL; 4924ab928f69208d240d3681336f34589e4b151824fSteve Wise break; 4934ab928f69208d240d3681336f34589e4b151824fSteve Wise } 494b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); 495b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe = (union t3_wr *) (qhp->wq.queue + idx); 496b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (num_wrs) 4974ab928f69208d240d3681336f34589e4b151824fSteve Wise if (wr->sg_list[0].lkey) 4984ab928f69208d240d3681336f34589e4b151824fSteve Wise err = build_rdma_recv(qhp, wqe, wr); 4994ab928f69208d240d3681336f34589e4b151824fSteve Wise else 5004ab928f69208d240d3681336f34589e4b151824fSteve Wise err = build_zero_stag_recv(qhp, wqe, wr); 501b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise else 502b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise err = -ENOMEM; 50348617f862f9e58ca2a609fea6a76733aff55d672Frank Zago 50448617f862f9e58ca2a609fea6a76733aff55d672Frank Zago if (err) 505b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 50648617f862f9e58ca2a609fea6a76733aff55d672Frank Zago 507b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise build_fw_riwrh((void *) wqe, T3_WR_RCV, T3_COMPLETION_FLAG, 508b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 509e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise 0, sizeof(struct t3_receive_wr) >> 3, T3_SOPEOP); 510b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s cookie 0x%llx idx 0x%x rq_wptr 0x%x rw_rptr 0x%x " 5113371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison "wqe %p \n", __func__, (unsigned long long) wr->wr_id, 512b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise idx, qhp->wq.rq_wptr, qhp->wq.rq_rptr, wqe); 513b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ++(qhp->wq.rq_wptr); 514b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ++(qhp->wq.wptr); 515b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wr = wr->next; 516b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise num_wrs--; 517b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 518b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 519e998f245c4b2d36ae2c35446e54ccbf1fb29d9deSteve Wise if (cxio_wq_db_enabled(&qhp->wq)) 520e998f245c4b2d36ae2c35446e54ccbf1fb29d9deSteve Wise ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); 52148617f862f9e58ca2a609fea6a76733aff55d672Frank Zago 52248617f862f9e58ca2a609fea6a76733aff55d672Frank Zagoout: 52348617f862f9e58ca2a609fea6a76733aff55d672Frank Zago if (err) 52448617f862f9e58ca2a609fea6a76733aff55d672Frank Zago *bad_wr = wr; 525b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return err; 526b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 527b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 528b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wiseint iwch_bind_mw(struct ib_qp *qp, 529b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct ib_mw *mw, 530b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct ib_mw_bind *mw_bind) 531b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 532b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_dev *rhp; 533b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_mw *mhp; 534b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_qp *qhp; 535b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise union t3_wr *wqe; 536b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 pbl_addr; 537b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u8 page_size; 538b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 num_wrs; 539b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise unsigned long flag; 540b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct ib_sge sgl; 541b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int err=0; 542b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise enum t3_wr_flags t3_wr_flags; 543b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 idx; 544b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct t3_swsq *sqp; 545b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 546b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp = to_iwch_qp(qp); 547b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp = to_iwch_mw(mw); 548b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise rhp = qhp->rhp; 549b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 550b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_lock_irqsave(&qhp->lock, flag); 551b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (qhp->attr.state > IWCH_QP_STATE_RTS) { 552b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 553b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 554b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 555b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, 556b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->wq.sq_size_log2); 5573d4f9a28e0f543e2a633d54f0f37f6e81a7701cdDan Carpenter if (num_wrs == 0) { 558b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 559b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -ENOMEM; 560b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 561b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); 5623371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison PDBG("%s: idx 0x%0x, mw 0x%p, mw_bind 0x%p\n", __func__, idx, 563b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mw, mw_bind); 564b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe = (union t3_wr *) (qhp->wq.queue + idx); 565b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 566b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise t3_wr_flags = 0; 567b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (mw_bind->send_flags & IB_SEND_SIGNALED) 568b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise t3_wr_flags = T3_COMPLETION_FLAG; 569b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 5707083e42ee2ff43a11481e0e7211ec4f9ac68cb79Shani Michaeli sgl.addr = mw_bind->bind_info.addr; 5717083e42ee2ff43a11481e0e7211ec4f9ac68cb79Shani Michaeli sgl.lkey = mw_bind->bind_info.mr->lkey; 5727083e42ee2ff43a11481e0e7211ec4f9ac68cb79Shani Michaeli sgl.length = mw_bind->bind_info.length; 573b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->bind.reserved = 0; 574e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise wqe->bind.type = TPT_VATO; 575b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 576b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* TBD: check perms */ 5777083e42ee2ff43a11481e0e7211ec4f9ac68cb79Shani Michaeli wqe->bind.perms = iwch_ib_to_tpt_bind_access( 5787083e42ee2ff43a11481e0e7211ec4f9ac68cb79Shani Michaeli mw_bind->bind_info.mw_access_flags); 5797083e42ee2ff43a11481e0e7211ec4f9ac68cb79Shani Michaeli wqe->bind.mr_stag = cpu_to_be32(mw_bind->bind_info.mr->lkey); 580b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->bind.mw_stag = cpu_to_be32(mw->rkey); 5817083e42ee2ff43a11481e0e7211ec4f9ac68cb79Shani Michaeli wqe->bind.mw_len = cpu_to_be32(mw_bind->bind_info.length); 5827083e42ee2ff43a11481e0e7211ec4f9ac68cb79Shani Michaeli wqe->bind.mw_va = cpu_to_be64(mw_bind->bind_info.addr); 583b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise err = iwch_sgl2pbl_map(rhp, &sgl, 1, &pbl_addr, &page_size); 584b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (err) { 585b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 586e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise return err; 587b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 588b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.wrid.id0.hi = qhp->wq.sq_wptr; 589b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp = qhp->wq.sq + Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2); 590b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->wr_id = mw_bind->wr_id; 591b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->opcode = T3_BIND_MW; 592b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->sq_wptr = qhp->wq.sq_wptr; 593b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->complete = 0; 594b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->signaled = (mw_bind->send_flags & IB_SEND_SIGNALED); 595b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->bind.mr_pbl_addr = cpu_to_be32(pbl_addr); 596b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->bind.mr_pagesz = page_size; 597b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise build_fw_riwrh((void *)wqe, T3_WR_BIND, t3_wr_flags, 598b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 0, 599e7e55829999deaab3f43e201a087731c02c54cf9Steve Wise sizeof(struct t3_bind_mw_wr) >> 3, T3_SOPEOP); 600b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ++(qhp->wq.wptr); 601b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ++(qhp->wq.sq_wptr); 602b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 603b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 604e998f245c4b2d36ae2c35446e54ccbf1fb29d9deSteve Wise if (cxio_wq_db_enabled(&qhp->wq)) 605e998f245c4b2d36ae2c35446e54ccbf1fb29d9deSteve Wise ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); 606b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 607b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return err; 608b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 609b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 6104a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wisestatic inline void build_term_codes(struct respQ_msg_t *rsp_msg, 6114a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise u8 *layer_type, u8 *ecode) 612b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 6134a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise int status = TPT_ERR_INTERNAL_ERR; 6144a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise int tagged = 0; 6154a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise int opcode = -1; 6164a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise int rqtype = 0; 6174a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise int send_inv = 0; 6184a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise 6194a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise if (rsp_msg) { 6204a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise status = CQE_STATUS(rsp_msg->cqe); 6214a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise opcode = CQE_OPCODE(rsp_msg->cqe); 6224a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise rqtype = RQ_TYPE(rsp_msg->cqe); 6234a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise send_inv = (opcode == T3_SEND_WITH_INV) || 6244a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise (opcode == T3_SEND_WITH_SE_INV); 6254a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise tagged = (opcode == T3_RDMA_WRITE) || 6264a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise (rqtype && (opcode == T3_READ_RESP)); 6274a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise } 6284a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise 6294a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise switch (status) { 630b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_STAG: 6314a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise if (send_inv) { 6324a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 6334a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise *ecode = RDMAP_CANT_INV_STAG; 6344a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise } else { 635b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 636b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = RDMAP_INV_STAG; 637b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 638b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 639b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_PDID: 6404a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 6414a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise if ((opcode == T3_SEND_WITH_INV) || 6424a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise (opcode == T3_SEND_WITH_SE_INV)) 6434a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise *ecode = RDMAP_CANT_INV_STAG; 6444a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise else 6454a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise *ecode = RDMAP_STAG_NOT_ASSOC; 6464a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise break; 647b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_QPID: 6484a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 6494a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise *ecode = RDMAP_STAG_NOT_ASSOC; 6504a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise break; 651b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_ACCESS: 6524a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 6534a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise *ecode = RDMAP_ACC_VIOL; 654b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 655b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_WRAP: 656b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 657b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = RDMAP_TO_WRAP; 658b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 659b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_BOUND: 6604a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise if (tagged) { 661b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_DDP|DDP_TAGGED_ERR; 662b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = DDPT_BASE_BOUNDS; 6634a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise } else { 664b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 665b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = RDMAP_BASE_BOUNDS; 666b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 667b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 668b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_INVALIDATE_SHARED_MR: 669b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND: 670b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 671b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = RDMAP_CANT_INV_STAG; 672b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 673b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_ECC: 674b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_ECC_PSTAG: 675b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_INTERNAL_ERR: 676b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_RDMAP|RDMAP_LOCAL_CATA; 677b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = 0; 678b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 679b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_OUT_OF_RQE: 680b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 681b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = DDPU_INV_MSN_NOBUF; 682b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 683b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_PBL_ADDR_BOUND: 684b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_DDP|DDP_TAGGED_ERR; 685b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = DDPT_BASE_BOUNDS; 686b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 687b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_CRC: 688b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_MPA|DDP_LLP; 689b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = MPA_CRC_ERR; 690b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 691b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_MARKER: 692b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_MPA|DDP_LLP; 693b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = MPA_MARKER_ERR; 694b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 695b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_PDU_LEN_ERR: 696b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 697b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = DDPU_MSG_TOOBIG; 698b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 699b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_DDP_VERSION: 700b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (tagged) { 701b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_DDP|DDP_TAGGED_ERR; 702b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = DDPT_INV_VERS; 703b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } else { 704b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 705b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = DDPU_INV_VERS; 706b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 707b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 708b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_RDMA_VERSION: 709b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 710b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = RDMAP_INV_VERS; 711b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 712b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_OPCODE: 713b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 714b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = RDMAP_INV_OPCODE; 715b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 716b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_DDP_QUEUE_NUM: 717b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 718b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = DDPU_INV_QN; 719b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 720b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_MSN: 721b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_MSN_GAP: 722b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_MSN_RANGE: 723b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_IRD_OVERFLOW: 724b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 725b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = DDPU_INV_MSN_RANGE; 726b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 727b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_TBIT: 728b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_DDP|DDP_LOCAL_CATA; 729b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = 0; 730b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 731b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_MO: 732b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 733b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = DDPU_INV_MO; 734b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 735b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise default: 736b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_RDMAP|DDP_LOCAL_CATA; 737b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = 0; 738b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 739b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 740b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 741b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 742807838686eb9e40d73b8a3f2384881358f51fff0Steve Wiseint iwch_post_zb_read(struct iwch_ep *ep) 743f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise{ 744f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise union t3_wr *wqe; 745f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise struct sk_buff *skb; 746f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise u8 flit_cnt = sizeof(struct t3_rdma_read_wr) >> 3; 747f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise 748f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise PDBG("%s enter\n", __func__); 749f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise skb = alloc_skb(40, GFP_KERNEL); 750f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise if (!skb) { 751f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise printk(KERN_ERR "%s cannot send zb_read!!\n", __func__); 752f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise return -ENOMEM; 753f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise } 754f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise wqe = (union t3_wr *)skb_put(skb, sizeof(struct t3_rdma_read_wr)); 755f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise memset(wqe, 0, sizeof(struct t3_rdma_read_wr)); 756f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise wqe->read.rdmaop = T3_READ_REQ; 757f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise wqe->read.reserved[0] = 0; 758f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise wqe->read.reserved[1] = 0; 759f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise wqe->read.rem_stag = cpu_to_be32(1); 760f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise wqe->read.rem_to = cpu_to_be64(1); 761f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise wqe->read.local_stag = cpu_to_be32(1); 762f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise wqe->read.local_len = cpu_to_be32(0); 763f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise wqe->read.local_to = cpu_to_be64(1); 764f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise wqe->send.wrh.op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(T3_WR_READ)); 765807838686eb9e40d73b8a3f2384881358f51fff0Steve Wise wqe->send.wrh.gen_tid_len = cpu_to_be32(V_FW_RIWR_TID(ep->hwtid)| 766f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise V_FW_RIWR_LEN(flit_cnt)); 767f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise skb->priority = CPL_PRIORITY_DATA; 768807838686eb9e40d73b8a3f2384881358f51fff0Steve Wise return iwch_cxgb3_ofld_send(ep->com.qp->rhp->rdev.t3cdev_p, skb); 769f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise} 770f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise 771b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise/* 772b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * This posts a TERMINATE with layer=RDMA, type=catastrophic. 773b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise */ 774b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wiseint iwch_post_terminate(struct iwch_qp *qhp, struct respQ_msg_t *rsp_msg) 775b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 776b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise union t3_wr *wqe; 777b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct terminate_message *term; 778b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct sk_buff *skb; 779b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 7803371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison PDBG("%s %d\n", __func__, __LINE__); 781b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise skb = alloc_skb(40, GFP_ATOMIC); 782b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!skb) { 7833371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison printk(KERN_ERR "%s cannot send TERMINATE!\n", __func__); 784b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -ENOMEM; 785b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 786b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe = (union t3_wr *)skb_put(skb, 40); 787b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise memset(wqe, 0, 40); 788b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.rdmaop = T3_TERMINATE; 789b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 790b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* immediate data length */ 791b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.plen = htonl(4); 792b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 793b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* immediate data starts here. */ 794b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise term = (struct terminate_message *)wqe->send.sgl; 7954a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise build_term_codes(rsp_msg, &term->layer_etype, &term->ecode); 796fb497d726612bb0e50240405aaabcf37f13901edSteve Wise wqe->send.wrh.op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(T3_WR_SEND) | 797fb497d726612bb0e50240405aaabcf37f13901edSteve Wise V_FW_RIWR_FLAGS(T3_COMPLETION_FLAG | T3_NOTIFY_FLAG)); 798fb497d726612bb0e50240405aaabcf37f13901edSteve Wise wqe->send.wrh.gen_tid_len = cpu_to_be32(V_FW_RIWR_TID(qhp->ep->hwtid)); 799b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise skb->priority = CPL_PRIORITY_DATA; 80004b5d028f50ff05a8f9ae049ee71f8fdfcf1f5deSteve Wise return iwch_cxgb3_ofld_send(qhp->rhp->rdev.t3cdev_p, skb); 801b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 802b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 803b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise/* 804b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * Assumes qhp lock is held. 805b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise */ 806b955150ea784af4c193b708a2e8091673bf23004Steve Wisestatic void __flush_qp(struct iwch_qp *qhp, struct iwch_cq *rchp, 807db4106ce635830201fad1bfca731a635beab6a72Steve Wise struct iwch_cq *schp) 808b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 809b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int count; 810c8286944b802c5ce4063ec3c334b38c6757a9434Steve Wise int flushed; 811b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 812b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 8133371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison PDBG("%s qhp %p rchp %p schp %p\n", __func__, qhp, rchp, schp); 814b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* take a ref on the qhp since we must release the lock */ 815b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise atomic_inc(&qhp->refcnt); 816db4106ce635830201fad1bfca731a635beab6a72Steve Wise spin_unlock(&qhp->lock); 817b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 818732bee7af3102cad811fb047dee8d15966efe569Uwe Kleine-König /* locking hierarchy: cq lock first, then qp lock. */ 819db4106ce635830201fad1bfca731a635beab6a72Steve Wise spin_lock(&rchp->lock); 820b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_lock(&qhp->lock); 821b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise cxio_flush_hw_cq(&rchp->cq); 822b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise cxio_count_rcqes(&rchp->cq, &qhp->wq, &count); 823c8286944b802c5ce4063ec3c334b38c6757a9434Steve Wise flushed = cxio_flush_rq(&qhp->wq, &rchp->cq, count); 824b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock(&qhp->lock); 825db4106ce635830201fad1bfca731a635beab6a72Steve Wise spin_unlock(&rchp->lock); 826f7cc25d018f1e9af6767ee7774bbe83452e9fdf4Kumar Sanghvi if (flushed) { 827db4106ce635830201fad1bfca731a635beab6a72Steve Wise spin_lock(&rchp->comp_handler_lock); 828c8286944b802c5ce4063ec3c334b38c6757a9434Steve Wise (*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context); 829db4106ce635830201fad1bfca731a635beab6a72Steve Wise spin_unlock(&rchp->comp_handler_lock); 830f7cc25d018f1e9af6767ee7774bbe83452e9fdf4Kumar Sanghvi } 831b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 832732bee7af3102cad811fb047dee8d15966efe569Uwe Kleine-König /* locking hierarchy: cq lock first, then qp lock. */ 833db4106ce635830201fad1bfca731a635beab6a72Steve Wise spin_lock(&schp->lock); 834b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_lock(&qhp->lock); 835b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise cxio_flush_hw_cq(&schp->cq); 836b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise cxio_count_scqes(&schp->cq, &qhp->wq, &count); 837c8286944b802c5ce4063ec3c334b38c6757a9434Steve Wise flushed = cxio_flush_sq(&qhp->wq, &schp->cq, count); 838b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock(&qhp->lock); 839db4106ce635830201fad1bfca731a635beab6a72Steve Wise spin_unlock(&schp->lock); 840f7cc25d018f1e9af6767ee7774bbe83452e9fdf4Kumar Sanghvi if (flushed) { 841db4106ce635830201fad1bfca731a635beab6a72Steve Wise spin_lock(&schp->comp_handler_lock); 842c8286944b802c5ce4063ec3c334b38c6757a9434Steve Wise (*schp->ibcq.comp_handler)(&schp->ibcq, schp->ibcq.cq_context); 843db4106ce635830201fad1bfca731a635beab6a72Steve Wise spin_unlock(&schp->comp_handler_lock); 844f7cc25d018f1e9af6767ee7774bbe83452e9fdf4Kumar Sanghvi } 845b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 846b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* deref */ 847b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (atomic_dec_and_test(&qhp->refcnt)) 848b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wake_up(&qhp->wait); 849b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 850db4106ce635830201fad1bfca731a635beab6a72Steve Wise spin_lock(&qhp->lock); 851b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 852b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 853db4106ce635830201fad1bfca731a635beab6a72Steve Wisestatic void flush_qp(struct iwch_qp *qhp) 854b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 855b955150ea784af4c193b708a2e8091673bf23004Steve Wise struct iwch_cq *rchp, *schp; 856b955150ea784af4c193b708a2e8091673bf23004Steve Wise 857b955150ea784af4c193b708a2e8091673bf23004Steve Wise rchp = get_chp(qhp->rhp, qhp->attr.rcq); 858b955150ea784af4c193b708a2e8091673bf23004Steve Wise schp = get_chp(qhp->rhp, qhp->attr.scq); 859b955150ea784af4c193b708a2e8091673bf23004Steve Wise 860b955150ea784af4c193b708a2e8091673bf23004Steve Wise if (qhp->ibqp.uobject) { 861b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise cxio_set_wq_in_error(&qhp->wq); 862b955150ea784af4c193b708a2e8091673bf23004Steve Wise cxio_set_cq_in_error(&rchp->cq); 863db4106ce635830201fad1bfca731a635beab6a72Steve Wise spin_lock(&rchp->comp_handler_lock); 864b955150ea784af4c193b708a2e8091673bf23004Steve Wise (*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context); 865db4106ce635830201fad1bfca731a635beab6a72Steve Wise spin_unlock(&rchp->comp_handler_lock); 866b955150ea784af4c193b708a2e8091673bf23004Steve Wise if (schp != rchp) { 867b955150ea784af4c193b708a2e8091673bf23004Steve Wise cxio_set_cq_in_error(&schp->cq); 868db4106ce635830201fad1bfca731a635beab6a72Steve Wise spin_lock(&schp->comp_handler_lock); 869b955150ea784af4c193b708a2e8091673bf23004Steve Wise (*schp->ibcq.comp_handler)(&schp->ibcq, 870b955150ea784af4c193b708a2e8091673bf23004Steve Wise schp->ibcq.cq_context); 871db4106ce635830201fad1bfca731a635beab6a72Steve Wise spin_unlock(&schp->comp_handler_lock); 872b955150ea784af4c193b708a2e8091673bf23004Steve Wise } 873b955150ea784af4c193b708a2e8091673bf23004Steve Wise return; 874b955150ea784af4c193b708a2e8091673bf23004Steve Wise } 875db4106ce635830201fad1bfca731a635beab6a72Steve Wise __flush_qp(qhp, rchp, schp); 876b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 877b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 878b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 879b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise/* 880f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise * Return count of RECV WRs posted 881b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise */ 882f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wiseu16 iwch_rqes_posted(struct iwch_qp *qhp) 883b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 884f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise union t3_wr *wqe = qhp->wq.queue; 885f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise u16 count = 0; 88621bfd4706268c85c9d17ca2fd48de3b19c9d40dbDan Carpenter 88721bfd4706268c85c9d17ca2fd48de3b19c9d40dbDan Carpenter while (count < USHRT_MAX && fw_riwrh_opcode((struct fw_riwrh *)wqe) == T3_WR_RCV) { 888f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise count++; 889f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise wqe++; 890f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise } 891f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise PDBG("%s qhp %p count %u\n", __func__, qhp, count); 892f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise return count; 893b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 894b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 895b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wisestatic int rdma_init(struct iwch_dev *rhp, struct iwch_qp *qhp, 896b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise enum iwch_qp_attr_mask mask, 897b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_qp_attributes *attrs) 898b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 899b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct t3_rdma_init_attr init_attr; 900b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int ret; 901b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 902b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.tid = qhp->ep->hwtid; 903b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.qpid = qhp->wq.qpid; 904b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.pdid = qhp->attr.pd; 905b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.scqid = qhp->attr.scq; 906b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.rcqid = qhp->attr.rcq; 907b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.rq_addr = qhp->wq.rq_addr; 908b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.rq_size = 1 << qhp->wq.rq_size_log2; 909b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.mpaattrs = uP_RI_MPA_IETF_ENABLE | 910b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.mpa_attr.recv_marker_enabled | 911b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (qhp->attr.mpa_attr.xmit_marker_enabled << 1) | 912b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (qhp->attr.mpa_attr.crc_enabled << 2); 913b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 9145f0f66b022ba607db0a083bf5cc13e4a4336e366Steve Wise init_attr.qpcaps = uP_RI_QP_RDMA_READ_ENABLE | 9155f0f66b022ba607db0a083bf5cc13e4a4336e366Steve Wise uP_RI_QP_RDMA_WRITE_ENABLE | 9165f0f66b022ba607db0a083bf5cc13e4a4336e366Steve Wise uP_RI_QP_BIND_ENABLE; 9175f0f66b022ba607db0a083bf5cc13e4a4336e366Steve Wise if (!qhp->ibqp.uobject) 9185f0f66b022ba607db0a083bf5cc13e4a4336e366Steve Wise init_attr.qpcaps |= uP_RI_QP_STAG0_ENABLE | 9195f0f66b022ba607db0a083bf5cc13e4a4336e366Steve Wise uP_RI_QP_FAST_REGISTER_ENABLE; 9205f0f66b022ba607db0a083bf5cc13e4a4336e366Steve Wise 921b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.tcp_emss = qhp->ep->emss; 922b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.ord = qhp->attr.max_ord; 923b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.ird = qhp->attr.max_ird; 924b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.qp_dma_addr = qhp->wq.dma_addr; 925b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.qp_dma_size = (1UL << qhp->wq.size_log2); 926f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise init_attr.rqe_count = iwch_rqes_posted(qhp); 927f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise init_attr.flags = qhp->attr.mpa_attr.initiator ? MPA_INITIATOR : 0; 928b496fe82d4075847a1c42efba2e81d28f6467b3aSteve Wise init_attr.chan = qhp->ep->l2t->smt_idx; 929f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise if (peer2peer) { 930f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise init_attr.rtr_type = RTR_READ; 931f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise if (init_attr.ord == 0 && qhp->attr.mpa_attr.initiator) 932f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise init_attr.ord = 1; 933f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise if (init_attr.ird == 0 && !qhp->attr.mpa_attr.initiator) 934f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise init_attr.ird = 1; 935f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise } else 936f8b0dfd15277974b5c9f3ff17f9e3ab6fdbe45eeSteve Wise init_attr.rtr_type = 0; 937de3d353072f9342f04112ba0504c3e294220cb8fSteve Wise init_attr.irs = qhp->ep->rcv_seq; 938b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s init_attr.rq_addr 0x%x init_attr.rq_size = %d " 9393371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison "flags 0x%x qpcaps 0x%x\n", __func__, 940b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.rq_addr, init_attr.rq_size, 941b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.flags, init_attr.qpcaps); 942b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = cxio_rdma_init(&rhp->rdev, &init_attr); 9433371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison PDBG("%s ret %d\n", __func__, ret); 944b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return ret; 945b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 946b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 947b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wiseint iwch_modify_qp(struct iwch_dev *rhp, struct iwch_qp *qhp, 948b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise enum iwch_qp_attr_mask mask, 949b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_qp_attributes *attrs, 950b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int internal) 951b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 952b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int ret = 0; 953b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_qp_attributes newattr = qhp->attr; 954b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise unsigned long flag; 955b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int disconnect = 0; 956b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int terminate = 0; 957b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int abort = 0; 958b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int free = 0; 959b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_ep *ep = NULL; 960b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 9613371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison PDBG("%s qhp %p qpid 0x%x ep %p state %d -> %d\n", __func__, 962b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp, qhp->wq.qpid, qhp->ep, qhp->attr.state, 963b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (mask & IWCH_QP_ATTR_NEXT_STATE) ? attrs->next_state : -1); 964b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 965b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_lock_irqsave(&qhp->lock, flag); 966b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 967b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* Process attr changes if in IDLE */ 968b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (mask & IWCH_QP_ATTR_VALID_MODIFY) { 969b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (qhp->attr.state != IWCH_QP_STATE_IDLE) { 970b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EIO; 971b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 972b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 973b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (mask & IWCH_QP_ATTR_ENABLE_RDMA_READ) 974b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise newattr.enable_rdma_read = attrs->enable_rdma_read; 975b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (mask & IWCH_QP_ATTR_ENABLE_RDMA_WRITE) 976b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise newattr.enable_rdma_write = attrs->enable_rdma_write; 977b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (mask & IWCH_QP_ATTR_ENABLE_RDMA_BIND) 978b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise newattr.enable_bind = attrs->enable_bind; 979b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (mask & IWCH_QP_ATTR_MAX_ORD) { 980b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (attrs->max_ord > 981b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise rhp->attr.max_rdma_read_qp_depth) { 982b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 983b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 984b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 985b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise newattr.max_ord = attrs->max_ord; 986b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 987b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (mask & IWCH_QP_ATTR_MAX_IRD) { 988b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (attrs->max_ird > 989b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise rhp->attr.max_rdma_reads_per_qp) { 990b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 991b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 992b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 993b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise newattr.max_ird = attrs->max_ird; 994b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 995b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr = newattr; 996b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 997b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 998b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!(mask & IWCH_QP_ATTR_NEXT_STATE)) 999b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 1000b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (qhp->attr.state == attrs->next_state) 1001b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 1002b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 1003b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise switch (qhp->attr.state) { 1004b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_IDLE: 1005b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise switch (attrs->next_state) { 1006b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_RTS: 1007b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!(mask & IWCH_QP_ATTR_LLP_STREAM_HANDLE)) { 1008b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 1009b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 1010b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 1011b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!(mask & IWCH_QP_ATTR_MPA_ATTR)) { 1012b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 1013b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 1014b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 1015b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.mpa_attr = attrs->mpa_attr; 1016b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.llp_stream_handle = attrs->llp_stream_handle; 1017b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->ep = qhp->attr.llp_stream_handle; 1018b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.state = IWCH_QP_STATE_RTS; 1019b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 1020b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* 1021b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * Ref the endpoint here and deref when we 1022b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * disassociate the endpoint from the QP. This 1023b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * happens in CLOSING->IDLE transition or *->ERROR 1024b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * transition. 1025b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise */ 1026b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise get_ep(&qhp->ep->com); 1027b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 1028b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = rdma_init(rhp, qhp, mask, attrs); 1029b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_lock_irqsave(&qhp->lock, flag); 1030b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (ret) 1031b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto err; 1032b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 1033b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_ERROR: 1034b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.state = IWCH_QP_STATE_ERROR; 1035db4106ce635830201fad1bfca731a635beab6a72Steve Wise flush_qp(qhp); 1036b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 1037b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise default: 1038b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 1039b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 1040b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 1041b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 1042b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_RTS: 1043b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise switch (attrs->next_state) { 1044b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_CLOSING: 1045b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise BUG_ON(atomic_read(&qhp->ep->com.kref.refcount) < 2); 1046b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.state = IWCH_QP_STATE_CLOSING; 1047b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!internal) { 1048b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise abort=0; 1049b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise disconnect = 1; 1050b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ep = qhp->ep; 1051989a1780698c65dfe093a6aa89ceeff84c31f528Steve Wise get_ep(&ep->com); 1052b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 1053b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 1054b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_TERMINATE: 1055b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.state = IWCH_QP_STATE_TERMINATE; 1056856b5925047d73a85557203d124d62c5eea1fbd3Steve Wise if (qhp->ibqp.uobject) 1057a1a750523b58cc4bb5a94fbb275a6f2a8bd9ace7Steve Wise cxio_set_wq_in_error(&qhp->wq); 1058b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!internal) 1059b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise terminate = 1; 1060b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 1061b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_ERROR: 1062b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.state = IWCH_QP_STATE_ERROR; 1063b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!internal) { 1064b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise abort=1; 1065b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise disconnect = 1; 1066b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ep = qhp->ep; 1067989a1780698c65dfe093a6aa89ceeff84c31f528Steve Wise get_ep(&ep->com); 1068b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 1069b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto err; 1070b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 1071b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise default: 1072b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 1073b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 1074b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 1075b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 1076b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_CLOSING: 1077b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!internal) { 1078b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 1079b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 1080b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 1081b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise switch (attrs->next_state) { 1082b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_IDLE: 1083db4106ce635830201fad1bfca731a635beab6a72Steve Wise flush_qp(qhp); 1084b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.state = IWCH_QP_STATE_IDLE; 1085b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.llp_stream_handle = NULL; 1086b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise put_ep(&qhp->ep->com); 1087b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->ep = NULL; 1088b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wake_up(&qhp->wait); 1089b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 1090b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_ERROR: 1091b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto err; 1092b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise default: 1093b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 1094b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto err; 1095b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 1096b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 1097b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_ERROR: 1098b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (attrs->next_state != IWCH_QP_STATE_IDLE) { 1099b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 1100b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 1101b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 1102b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 1103b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!Q_EMPTY(qhp->wq.sq_rptr, qhp->wq.sq_wptr) || 1104b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise !Q_EMPTY(qhp->wq.rq_rptr, qhp->wq.rq_wptr)) { 1105b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 1106b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 1107b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 1108b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.state = IWCH_QP_STATE_IDLE; 1109b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 1110b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_TERMINATE: 1111b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!internal) { 1112b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 1113b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 1114b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 1115b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto err; 1116b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 1117b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise default: 1118b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise printk(KERN_ERR "%s in a bad state %d\n", 11193371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison __func__, qhp->attr.state); 1120b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 1121b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto err; 1122b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 1123b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 1124b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 1125b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wiseerr: 11263371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison PDBG("%s disassociating ep %p qpid 0x%x\n", __func__, qhp->ep, 1127b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->wq.qpid); 1128b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 1129b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* disassociate the LLP connection */ 1130b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.llp_stream_handle = NULL; 1131b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ep = qhp->ep; 1132b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->ep = NULL; 1133b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.state = IWCH_QP_STATE_ERROR; 1134b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise free=1; 1135b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wake_up(&qhp->wait); 1136b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise BUG_ON(!ep); 1137db4106ce635830201fad1bfca731a635beab6a72Steve Wise flush_qp(qhp); 1138b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wiseout: 1139b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 1140b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 1141b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (terminate) 1142b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise iwch_post_terminate(qhp, NULL); 1143b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 1144b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* 1145b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * If disconnect is 1, then we need to initiate a disconnect 1146b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * on the EP. This can be a normal close (RTS->CLOSING) or 1147b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * an abnormal close (RTS/CLOSING->ERROR). 1148b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise */ 1149989a1780698c65dfe093a6aa89ceeff84c31f528Steve Wise if (disconnect) { 1150b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise iwch_ep_disconnect(ep, abort, GFP_KERNEL); 1151989a1780698c65dfe093a6aa89ceeff84c31f528Steve Wise put_ep(&ep->com); 1152989a1780698c65dfe093a6aa89ceeff84c31f528Steve Wise } 1153b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 1154b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* 1155b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * If free is 1, then we've disassociated the EP from the QP 1156b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * and we need to dereference the EP. 1157b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise */ 1158b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (free) 1159b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise put_ep(&ep->com); 1160b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 11613371836383d63b627b228875f5ac63023cbf11d2Harvey Harrison PDBG("%s exit state %d\n", __func__, qhp->attr.state); 1162b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return ret; 1163b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 1164