iwch_qp.c revision de3d353072f9342f04112ba0504c3e294220cb8f
1b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise/* 2b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * Copyright (c) 2006 Chelsio, Inc. All rights reserved. 3b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * 4b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * This software is available to you under a choice of one of two 5b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * licenses. You may choose to be licensed under the terms of the GNU 6b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * General Public License (GPL) Version 2, available from the file 7b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * COPYING in the main directory of this source tree, or the 8b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * OpenIB.org BSD license below: 9b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * 10b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * Redistribution and use in source and binary forms, with or 11b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * without modification, are permitted provided that the following 12b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * conditions are met: 13b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * 14b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * - Redistributions of source code must retain the above 15b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * copyright notice, this list of conditions and the following 16b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * disclaimer. 17b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * 18b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * - Redistributions in binary form must reproduce the above 19b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * copyright notice, this list of conditions and the following 20b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * disclaimer in the documentation and/or other materials 21b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * provided with the distribution. 22b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * 23b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * SOFTWARE. 31b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise */ 32b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#include "iwch_provider.h" 33b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#include "iwch.h" 34b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#include "iwch_cm.h" 35b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#include "cxio_hal.h" 36b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 37b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#define NO_SUPPORT -1 38b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 392b540355cd2f46c5445030995e72c4b4fb2b775eAdrian Bunkstatic int iwch_build_rdma_send(union t3_wr *wqe, struct ib_send_wr *wr, 402b540355cd2f46c5445030995e72c4b4fb2b775eAdrian Bunk u8 * flit_cnt) 41b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 42b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int i; 43b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 plen; 44b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 45b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise switch (wr->opcode) { 46b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IB_WR_SEND: 47b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IB_WR_SEND_WITH_IMM: 48b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (wr->send_flags & IB_SEND_SOLICITED) 49b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.rdmaop = T3_SEND_WITH_SE; 50b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise else 51b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.rdmaop = T3_SEND; 52b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.rem_stag = 0; 53b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 54b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#if 0 /* Not currently supported */ 55b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TYPE_SEND_INVALIDATE: 56b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TYPE_SEND_INVALIDATE_IMMEDIATE: 57b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.rdmaop = T3_SEND_WITH_INV; 58b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.rem_stag = cpu_to_be32(wr->wr.rdma.rkey); 59b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 60b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TYPE_SEND_SE_INVALIDATE: 61b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.rdmaop = T3_SEND_WITH_SE_INV; 62b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.rem_stag = cpu_to_be32(wr->wr.rdma.rkey); 63b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 64b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#endif 65b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise default: 66b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 67b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 68b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (wr->num_sge > T3_MAX_SGE) 69b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 70b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.reserved[0] = 0; 71b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.reserved[1] = 0; 72b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.reserved[2] = 0; 73b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (wr->opcode == IB_WR_SEND_WITH_IMM) { 74b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise plen = 4; 75b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.sgl[0].stag = wr->imm_data; 76b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.sgl[0].len = __constant_cpu_to_be32(0); 77b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.num_sgle = __constant_cpu_to_be32(0); 78b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *flit_cnt = 5; 79b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } else { 80b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise plen = 0; 81b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise for (i = 0; i < wr->num_sge; i++) { 82b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if ((plen + wr->sg_list[i].length) < plen) { 83b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EMSGSIZE; 84b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 85b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise plen += wr->sg_list[i].length; 86b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.sgl[i].stag = 87b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise cpu_to_be32(wr->sg_list[i].lkey); 88b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.sgl[i].len = 89b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise cpu_to_be32(wr->sg_list[i].length); 90b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.sgl[i].to = cpu_to_be64(wr->sg_list[i].addr); 91b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 92b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.num_sgle = cpu_to_be32(wr->num_sge); 93b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *flit_cnt = 4 + ((wr->num_sge) << 1); 94b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 95b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.plen = cpu_to_be32(plen); 96b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return 0; 97b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 98b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 992b540355cd2f46c5445030995e72c4b4fb2b775eAdrian Bunkstatic int iwch_build_rdma_write(union t3_wr *wqe, struct ib_send_wr *wr, 1002b540355cd2f46c5445030995e72c4b4fb2b775eAdrian Bunk u8 *flit_cnt) 101b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 102b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int i; 103b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 plen; 104b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (wr->num_sge > T3_MAX_SGE) 105b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 106b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.rdmaop = T3_RDMA_WRITE; 107b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.reserved[0] = 0; 108b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.reserved[1] = 0; 109b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.reserved[2] = 0; 110b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.stag_sink = cpu_to_be32(wr->wr.rdma.rkey); 111b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.to_sink = cpu_to_be64(wr->wr.rdma.remote_addr); 112b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 113b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) { 114b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise plen = 4; 115b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.sgl[0].stag = wr->imm_data; 116b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.sgl[0].len = __constant_cpu_to_be32(0); 117b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.num_sgle = __constant_cpu_to_be32(0); 118b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *flit_cnt = 6; 119b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } else { 120b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise plen = 0; 121b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise for (i = 0; i < wr->num_sge; i++) { 122b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if ((plen + wr->sg_list[i].length) < plen) { 123b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EMSGSIZE; 124b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 125b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise plen += wr->sg_list[i].length; 126b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.sgl[i].stag = 127b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise cpu_to_be32(wr->sg_list[i].lkey); 128b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.sgl[i].len = 129b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise cpu_to_be32(wr->sg_list[i].length); 130b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.sgl[i].to = 131b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise cpu_to_be64(wr->sg_list[i].addr); 132b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 133b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.num_sgle = cpu_to_be32(wr->num_sge); 134b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *flit_cnt = 5 + ((wr->num_sge) << 1); 135b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 136b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->write.plen = cpu_to_be32(plen); 137b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return 0; 138b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 139b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 1402b540355cd2f46c5445030995e72c4b4fb2b775eAdrian Bunkstatic int iwch_build_rdma_read(union t3_wr *wqe, struct ib_send_wr *wr, 1412b540355cd2f46c5445030995e72c4b4fb2b775eAdrian Bunk u8 *flit_cnt) 142b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 143b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (wr->num_sge > 1) 144b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 145b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->read.rdmaop = T3_READ_REQ; 146b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->read.reserved[0] = 0; 147b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->read.reserved[1] = 0; 148b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->read.reserved[2] = 0; 149b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->read.rem_stag = cpu_to_be32(wr->wr.rdma.rkey); 150b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->read.rem_to = cpu_to_be64(wr->wr.rdma.remote_addr); 151b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->read.local_stag = cpu_to_be32(wr->sg_list[0].lkey); 152b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->read.local_len = cpu_to_be32(wr->sg_list[0].length); 153b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->read.local_to = cpu_to_be64(wr->sg_list[0].addr); 154b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *flit_cnt = sizeof(struct t3_rdma_read_wr) >> 3; 155b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return 0; 156b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 157b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 158b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise/* 159b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * TBD: this is going to be moved to firmware. Missing pdid/qpid check for now. 160b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise */ 1612b540355cd2f46c5445030995e72c4b4fb2b775eAdrian Bunkstatic int iwch_sgl2pbl_map(struct iwch_dev *rhp, struct ib_sge *sg_list, 1622b540355cd2f46c5445030995e72c4b4fb2b775eAdrian Bunk u32 num_sgle, u32 * pbl_addr, u8 * page_size) 163b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 164b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int i; 165b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_mr *mhp; 166b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 offset; 167b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise for (i = 0; i < num_sgle; i++) { 168b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 169b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp = get_mhp(rhp, (sg_list[i].lkey) >> 8); 170b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!mhp) { 171b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s %d\n", __FUNCTION__, __LINE__); 172b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EIO; 173b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 174b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!mhp->attr.state) { 175b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s %d\n", __FUNCTION__, __LINE__); 176b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EIO; 177b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 178b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (mhp->attr.zbva) { 179b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s %d\n", __FUNCTION__, __LINE__); 180b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EIO; 181b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 182b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 183b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (sg_list[i].addr < mhp->attr.va_fbo) { 184b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s %d\n", __FUNCTION__, __LINE__); 185b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 186b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 187b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (sg_list[i].addr + ((u64) sg_list[i].length) < 188b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sg_list[i].addr) { 189b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s %d\n", __FUNCTION__, __LINE__); 190b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 191b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 192b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (sg_list[i].addr + ((u64) sg_list[i].length) > 193b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp->attr.va_fbo + ((u64) mhp->attr.len)) { 194b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s %d\n", __FUNCTION__, __LINE__); 195b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 196b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 197b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise offset = sg_list[i].addr - mhp->attr.va_fbo; 198b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise offset += ((u32) mhp->attr.va_fbo) % 199b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (1UL << (12 + mhp->attr.page_size)); 200b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise pbl_addr[i] = ((mhp->attr.pbl_addr - 201b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise rhp->rdev.rnic_info.pbl_base) >> 3) + 202b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (offset >> (12 + mhp->attr.page_size)); 203b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise page_size[i] = mhp->attr.page_size; 204b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 205b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return 0; 206b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 207b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 2082b540355cd2f46c5445030995e72c4b4fb2b775eAdrian Bunkstatic int iwch_build_rdma_recv(struct iwch_dev *rhp, union t3_wr *wqe, 2092b540355cd2f46c5445030995e72c4b4fb2b775eAdrian Bunk struct ib_recv_wr *wr) 210b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 211b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int i, err = 0; 212b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 pbl_addr[4]; 213b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u8 page_size[4]; 214b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (wr->num_sge > T3_MAX_SGE) 215b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 216b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise err = iwch_sgl2pbl_map(rhp, wr->sg_list, wr->num_sge, pbl_addr, 217b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise page_size); 218b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (err) 219b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return err; 220b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->recv.pagesz[0] = page_size[0]; 221b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->recv.pagesz[1] = page_size[1]; 222b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->recv.pagesz[2] = page_size[2]; 223b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->recv.pagesz[3] = page_size[3]; 224b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->recv.num_sgle = cpu_to_be32(wr->num_sge); 225b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise for (i = 0; i < wr->num_sge; i++) { 226b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->recv.sgl[i].stag = cpu_to_be32(wr->sg_list[i].lkey); 227b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->recv.sgl[i].len = cpu_to_be32(wr->sg_list[i].length); 228b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 229b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* to in the WQE == the offset into the page */ 230b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->recv.sgl[i].to = cpu_to_be64(((u32) wr->sg_list[i].addr) % 231b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (1UL << (12 + page_size[i]))); 232b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 233b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* pbl_addr is the adapters address in the PBL */ 234b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->recv.pbl_addr[i] = cpu_to_be32(pbl_addr[i]); 235b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 236b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise for (; i < T3_MAX_SGE; i++) { 237b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->recv.sgl[i].stag = 0; 238b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->recv.sgl[i].len = 0; 239b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->recv.sgl[i].to = 0; 240b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->recv.pbl_addr[i] = 0; 241b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 242b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return 0; 243b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 244b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 245b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wiseint iwch_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 246b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct ib_send_wr **bad_wr) 247b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 248b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int err = 0; 249b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u8 t3_wr_flit_cnt; 250b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise enum t3_wr_opcode t3_wr_opcode = 0; 251b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise enum t3_wr_flags t3_wr_flags; 252b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_qp *qhp; 253b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 idx; 254b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise union t3_wr *wqe; 255b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 num_wrs; 256b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise unsigned long flag; 257b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct t3_swsq *sqp; 258b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 259b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp = to_iwch_qp(ibqp); 260b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_lock_irqsave(&qhp->lock, flag); 261b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (qhp->attr.state > IWCH_QP_STATE_RTS) { 262b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 263b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 264b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 265b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, 266b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->wq.sq_size_log2); 267b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (num_wrs <= 0) { 268b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 269b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -ENOMEM; 270b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 271b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise while (wr) { 272b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (num_wrs == 0) { 273b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise err = -ENOMEM; 274b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *bad_wr = wr; 275b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 276b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 277b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); 278b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe = (union t3_wr *) (qhp->wq.queue + idx); 279b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise t3_wr_flags = 0; 280b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (wr->send_flags & IB_SEND_SOLICITED) 281b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise t3_wr_flags |= T3_SOLICITED_EVENT_FLAG; 282b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (wr->send_flags & IB_SEND_FENCE) 283b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise t3_wr_flags |= T3_READ_FENCE_FLAG; 284b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (wr->send_flags & IB_SEND_SIGNALED) 285b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise t3_wr_flags |= T3_COMPLETION_FLAG; 286b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp = qhp->wq.sq + 287b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2); 288b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise switch (wr->opcode) { 289b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IB_WR_SEND: 290b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IB_WR_SEND_WITH_IMM: 291b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise t3_wr_opcode = T3_WR_SEND; 292b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise err = iwch_build_rdma_send(wqe, wr, &t3_wr_flit_cnt); 293b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 294b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IB_WR_RDMA_WRITE: 295b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IB_WR_RDMA_WRITE_WITH_IMM: 296b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise t3_wr_opcode = T3_WR_WRITE; 297b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise err = iwch_build_rdma_write(wqe, wr, &t3_wr_flit_cnt); 298b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 299b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IB_WR_RDMA_READ: 300b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise t3_wr_opcode = T3_WR_READ; 301b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise t3_wr_flags = 0; /* T3 reads are always signaled */ 302b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise err = iwch_build_rdma_read(wqe, wr, &t3_wr_flit_cnt); 303b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (err) 304b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 305b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->read_len = wqe->read.local_len; 306b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!qhp->wq.oldest_read) 307b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->wq.oldest_read = sqp; 308b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 309b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise default: 310b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s post of type=%d TBD!\n", __FUNCTION__, 311b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wr->opcode); 312b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise err = -EINVAL; 313b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 314b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (err) { 315b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *bad_wr = wr; 316b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 317b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 318b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.wrid.id0.hi = qhp->wq.sq_wptr; 319b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->wr_id = wr->wr_id; 320b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->opcode = wr2opcode(t3_wr_opcode); 321b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->sq_wptr = qhp->wq.sq_wptr; 322b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->complete = 0; 323b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->signaled = (wr->send_flags & IB_SEND_SIGNALED); 324b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 325b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise build_fw_riwrh((void *) wqe, t3_wr_opcode, t3_wr_flags, 326b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 327b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 0, t3_wr_flit_cnt); 328b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s cookie 0x%llx wq idx 0x%x swsq idx %ld opcode %d\n", 329b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise __FUNCTION__, (unsigned long long) wr->wr_id, idx, 330b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2), 331b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->opcode); 332b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wr = wr->next; 333b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise num_wrs--; 334b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ++(qhp->wq.wptr); 335b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ++(qhp->wq.sq_wptr); 336b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 337b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 338b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); 339b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return err; 340b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 341b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 342b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wiseint iwch_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, 343b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct ib_recv_wr **bad_wr) 344b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 345b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int err = 0; 346b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_qp *qhp; 347b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 idx; 348b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise union t3_wr *wqe; 349b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 num_wrs; 350b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise unsigned long flag; 351b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 352b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp = to_iwch_qp(ibqp); 353b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_lock_irqsave(&qhp->lock, flag); 354b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (qhp->attr.state > IWCH_QP_STATE_RTS) { 355b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 356b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 357b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 358b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise num_wrs = Q_FREECNT(qhp->wq.rq_rptr, qhp->wq.rq_wptr, 359b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->wq.rq_size_log2) - 1; 360b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!wr) { 361b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 362b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 363b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 364b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise while (wr) { 365b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); 366b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe = (union t3_wr *) (qhp->wq.queue + idx); 367b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (num_wrs) 368b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise err = iwch_build_rdma_recv(qhp->rhp, wqe, wr); 369b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise else 370b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise err = -ENOMEM; 371b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (err) { 372b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *bad_wr = wr; 373b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 374b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 375b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr, qhp->wq.rq_size_log2)] = 376b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wr->wr_id; 377b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise build_fw_riwrh((void *) wqe, T3_WR_RCV, T3_COMPLETION_FLAG, 378b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 379b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 0, sizeof(struct t3_receive_wr) >> 3); 380b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s cookie 0x%llx idx 0x%x rq_wptr 0x%x rw_rptr 0x%x " 381b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise "wqe %p \n", __FUNCTION__, (unsigned long long) wr->wr_id, 382b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise idx, qhp->wq.rq_wptr, qhp->wq.rq_rptr, wqe); 383b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ++(qhp->wq.rq_wptr); 384b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ++(qhp->wq.wptr); 385b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wr = wr->next; 386b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise num_wrs--; 387b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 388b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 389b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); 390b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return err; 391b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 392b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 393b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wiseint iwch_bind_mw(struct ib_qp *qp, 394b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct ib_mw *mw, 395b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct ib_mw_bind *mw_bind) 396b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 397b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_dev *rhp; 398b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_mw *mhp; 399b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_qp *qhp; 400b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise union t3_wr *wqe; 401b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 pbl_addr; 402b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u8 page_size; 403b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 num_wrs; 404b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise unsigned long flag; 405b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct ib_sge sgl; 406b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int err=0; 407b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise enum t3_wr_flags t3_wr_flags; 408b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise u32 idx; 409b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct t3_swsq *sqp; 410b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 411b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp = to_iwch_qp(qp); 412b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mhp = to_iwch_mw(mw); 413b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise rhp = qhp->rhp; 414b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 415b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_lock_irqsave(&qhp->lock, flag); 416b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (qhp->attr.state > IWCH_QP_STATE_RTS) { 417b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 418b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -EINVAL; 419b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 420b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, 421b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->wq.sq_size_log2); 422b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if ((num_wrs) <= 0) { 423b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 424b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -ENOMEM; 425b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 426b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); 427b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s: idx 0x%0x, mw 0x%p, mw_bind 0x%p\n", __FUNCTION__, idx, 428b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise mw, mw_bind); 429b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe = (union t3_wr *) (qhp->wq.queue + idx); 430b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 431b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise t3_wr_flags = 0; 432b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (mw_bind->send_flags & IB_SEND_SIGNALED) 433b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise t3_wr_flags = T3_COMPLETION_FLAG; 434b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 435b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sgl.addr = mw_bind->addr; 436b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sgl.lkey = mw_bind->mr->lkey; 437b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sgl.length = mw_bind->length; 438b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->bind.reserved = 0; 439b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->bind.type = T3_VA_BASED_TO; 440b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 441b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* TBD: check perms */ 442e64518f3736971f22d5c22a6bab40891d2db337fSteve Wise wqe->bind.perms = iwch_ib_to_mwbind_access(mw_bind->mw_access_flags); 443b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->bind.mr_stag = cpu_to_be32(mw_bind->mr->lkey); 444b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->bind.mw_stag = cpu_to_be32(mw->rkey); 445b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->bind.mw_len = cpu_to_be32(mw_bind->length); 446b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->bind.mw_va = cpu_to_be64(mw_bind->addr); 447b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise err = iwch_sgl2pbl_map(rhp, &sgl, 1, &pbl_addr, &page_size); 448b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (err) { 449b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 450b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return err; 451b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 452b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.wrid.id0.hi = qhp->wq.sq_wptr; 453b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp = qhp->wq.sq + Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2); 454b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->wr_id = mw_bind->wr_id; 455b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->opcode = T3_BIND_MW; 456b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->sq_wptr = qhp->wq.sq_wptr; 457b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->complete = 0; 458b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sqp->signaled = (mw_bind->send_flags & IB_SEND_SIGNALED); 459b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->bind.mr_pbl_addr = cpu_to_be32(pbl_addr); 460b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->bind.mr_pagesz = page_size; 461b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->flit[T3_SQ_COOKIE_FLIT] = mw_bind->wr_id; 462b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise build_fw_riwrh((void *)wqe, T3_WR_BIND, t3_wr_flags, 463b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 0, 464b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise sizeof(struct t3_bind_mw_wr) >> 3); 465b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ++(qhp->wq.wptr); 466b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ++(qhp->wq.sq_wptr); 467b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 468b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 469b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); 470b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 471b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return err; 472b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 473b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 4744a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wisestatic inline void build_term_codes(struct respQ_msg_t *rsp_msg, 4754a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise u8 *layer_type, u8 *ecode) 476b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 4774a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise int status = TPT_ERR_INTERNAL_ERR; 4784a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise int tagged = 0; 4794a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise int opcode = -1; 4804a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise int rqtype = 0; 4814a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise int send_inv = 0; 4824a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise 4834a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise if (rsp_msg) { 4844a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise status = CQE_STATUS(rsp_msg->cqe); 4854a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise opcode = CQE_OPCODE(rsp_msg->cqe); 4864a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise rqtype = RQ_TYPE(rsp_msg->cqe); 4874a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise send_inv = (opcode == T3_SEND_WITH_INV) || 4884a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise (opcode == T3_SEND_WITH_SE_INV); 4894a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise tagged = (opcode == T3_RDMA_WRITE) || 4904a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise (rqtype && (opcode == T3_READ_RESP)); 4914a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise } 4924a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise 4934a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise switch (status) { 494b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_STAG: 4954a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise if (send_inv) { 4964a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 4974a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise *ecode = RDMAP_CANT_INV_STAG; 4984a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise } else { 499b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 500b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = RDMAP_INV_STAG; 501b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 502b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 503b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_PDID: 5044a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 5054a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise if ((opcode == T3_SEND_WITH_INV) || 5064a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise (opcode == T3_SEND_WITH_SE_INV)) 5074a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise *ecode = RDMAP_CANT_INV_STAG; 5084a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise else 5094a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise *ecode = RDMAP_STAG_NOT_ASSOC; 5104a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise break; 511b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_QPID: 5124a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 5134a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise *ecode = RDMAP_STAG_NOT_ASSOC; 5144a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise break; 515b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_ACCESS: 5164a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 5174a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise *ecode = RDMAP_ACC_VIOL; 518b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 519b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_WRAP: 520b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 521b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = RDMAP_TO_WRAP; 522b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 523b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_BOUND: 5244a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise if (tagged) { 525b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_DDP|DDP_TAGGED_ERR; 526b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = DDPT_BASE_BOUNDS; 5274a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise } else { 528b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 529b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = RDMAP_BASE_BOUNDS; 530b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 531b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 532b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_INVALIDATE_SHARED_MR: 533b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND: 534b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 535b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = RDMAP_CANT_INV_STAG; 536b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 537b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_ECC: 538b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_ECC_PSTAG: 539b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_INTERNAL_ERR: 540b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_RDMAP|RDMAP_LOCAL_CATA; 541b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = 0; 542b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 543b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_OUT_OF_RQE: 544b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 545b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = DDPU_INV_MSN_NOBUF; 546b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 547b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_PBL_ADDR_BOUND: 548b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_DDP|DDP_TAGGED_ERR; 549b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = DDPT_BASE_BOUNDS; 550b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 551b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_CRC: 552b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_MPA|DDP_LLP; 553b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = MPA_CRC_ERR; 554b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 555b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_MARKER: 556b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_MPA|DDP_LLP; 557b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = MPA_MARKER_ERR; 558b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 559b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_PDU_LEN_ERR: 560b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 561b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = DDPU_MSG_TOOBIG; 562b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 563b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_DDP_VERSION: 564b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (tagged) { 565b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_DDP|DDP_TAGGED_ERR; 566b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = DDPT_INV_VERS; 567b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } else { 568b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 569b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = DDPU_INV_VERS; 570b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 571b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 572b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_RDMA_VERSION: 573b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 574b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = RDMAP_INV_VERS; 575b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 576b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_OPCODE: 577b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 578b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = RDMAP_INV_OPCODE; 579b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 580b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_DDP_QUEUE_NUM: 581b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 582b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = DDPU_INV_QN; 583b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 584b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_MSN: 585b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_MSN_GAP: 586b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_MSN_RANGE: 587b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_IRD_OVERFLOW: 588b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 589b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = DDPU_INV_MSN_RANGE; 590b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 591b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_TBIT: 592b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_DDP|DDP_LOCAL_CATA; 593b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = 0; 594b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 595b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case TPT_ERR_MO: 596b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 597b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = DDPU_INV_MO; 598b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 599b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise default: 600b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *layer_type = LAYER_RDMAP|DDP_LOCAL_CATA; 601b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise *ecode = 0; 602b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 603b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 604b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 605b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 606b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise/* 607b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * This posts a TERMINATE with layer=RDMA, type=catastrophic. 608b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise */ 609b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wiseint iwch_post_terminate(struct iwch_qp *qhp, struct respQ_msg_t *rsp_msg) 610b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 611b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise union t3_wr *wqe; 612b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct terminate_message *term; 613b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct sk_buff *skb; 614b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 615b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s %d\n", __FUNCTION__, __LINE__); 616b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise skb = alloc_skb(40, GFP_ATOMIC); 617b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!skb) { 618b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise printk(KERN_ERR "%s cannot send TERMINATE!\n", __FUNCTION__); 619b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return -ENOMEM; 620b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 621b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe = (union t3_wr *)skb_put(skb, 40); 622b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise memset(wqe, 0, 40); 623b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.rdmaop = T3_TERMINATE; 624b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 625b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* immediate data length */ 626b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wqe->send.plen = htonl(4); 627b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 628b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* immediate data starts here. */ 629b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise term = (struct terminate_message *)wqe->send.sgl; 6304a97d47ef7946cf31b76945c3199b0b5cad6a8edSteve Wise build_term_codes(rsp_msg, &term->layer_etype, &term->ecode); 631b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise build_fw_riwrh((void *)wqe, T3_WR_SEND, 632b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise T3_COMPLETION_FLAG | T3_NOTIFY_FLAG, 1, 633b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->ep->hwtid, 5); 634b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise skb->priority = CPL_PRIORITY_DATA; 635b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return cxgb3_ofld_send(qhp->rhp->rdev.t3cdev_p, skb); 636b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 637b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 638b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise/* 639b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * Assumes qhp lock is held. 640b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise */ 641b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wisestatic void __flush_qp(struct iwch_qp *qhp, unsigned long *flag) 642b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 643b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_cq *rchp, *schp; 644b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int count; 645b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 646b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise rchp = get_chp(qhp->rhp, qhp->attr.rcq); 647b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise schp = get_chp(qhp->rhp, qhp->attr.scq); 648b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 649b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s qhp %p rchp %p schp %p\n", __FUNCTION__, qhp, rchp, schp); 650b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* take a ref on the qhp since we must release the lock */ 651b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise atomic_inc(&qhp->refcnt); 652b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, *flag); 653b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 654b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* locking heirarchy: cq lock first, then qp lock. */ 655b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_lock_irqsave(&rchp->lock, *flag); 656b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_lock(&qhp->lock); 657b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise cxio_flush_hw_cq(&rchp->cq); 658b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise cxio_count_rcqes(&rchp->cq, &qhp->wq, &count); 659b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise cxio_flush_rq(&qhp->wq, &rchp->cq, count); 660b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock(&qhp->lock); 661b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&rchp->lock, *flag); 662b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 663b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* locking heirarchy: cq lock first, then qp lock. */ 664b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_lock_irqsave(&schp->lock, *flag); 665b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_lock(&qhp->lock); 666b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise cxio_flush_hw_cq(&schp->cq); 667b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise cxio_count_scqes(&schp->cq, &qhp->wq, &count); 668b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise cxio_flush_sq(&qhp->wq, &schp->cq, count); 669b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock(&qhp->lock); 670b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&schp->lock, *flag); 671b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 672b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* deref */ 673b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (atomic_dec_and_test(&qhp->refcnt)) 674b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wake_up(&qhp->wait); 675b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 676b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_lock_irqsave(&qhp->lock, *flag); 677b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 678b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 6792b540355cd2f46c5445030995e72c4b4fb2b775eAdrian Bunkstatic void flush_qp(struct iwch_qp *qhp, unsigned long *flag) 680b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 681b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (t3b_device(qhp->rhp)) 682b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise cxio_set_wq_in_error(&qhp->wq); 683b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise else 684b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise __flush_qp(qhp, flag); 685b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 686b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 687b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 688b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise/* 689b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * Return non zero if at least one RECV was pre-posted. 690b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise */ 6912b540355cd2f46c5445030995e72c4b4fb2b775eAdrian Bunkstatic int rqes_posted(struct iwch_qp *qhp) 692b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 693b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return fw_riwrh_opcode((struct fw_riwrh *)qhp->wq.queue) == T3_WR_RCV; 694b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 695b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 696b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wisestatic int rdma_init(struct iwch_dev *rhp, struct iwch_qp *qhp, 697b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise enum iwch_qp_attr_mask mask, 698b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_qp_attributes *attrs) 699b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 700b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct t3_rdma_init_attr init_attr; 701b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int ret; 702b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 703b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.tid = qhp->ep->hwtid; 704b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.qpid = qhp->wq.qpid; 705b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.pdid = qhp->attr.pd; 706b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.scqid = qhp->attr.scq; 707b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.rcqid = qhp->attr.rcq; 708b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.rq_addr = qhp->wq.rq_addr; 709b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.rq_size = 1 << qhp->wq.rq_size_log2; 710b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.mpaattrs = uP_RI_MPA_IETF_ENABLE | 711b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.mpa_attr.recv_marker_enabled | 712b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (qhp->attr.mpa_attr.xmit_marker_enabled << 1) | 713b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (qhp->attr.mpa_attr.crc_enabled << 2); 714b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 715b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* 716b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * XXX - The IWCM doesn't quite handle getting these 717b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * attrs set before going into RTS. For now, just turn 718b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * them on always... 719b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise */ 720b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#if 0 721b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.qpcaps = qhp->attr.enableRdmaRead | 722b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (qhp->attr.enableRdmaWrite << 1) | 723b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (qhp->attr.enableBind << 2) | 724b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (qhp->attr.enable_stag0_fastreg << 3) | 725b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (qhp->attr.enable_stag0_fastreg << 4); 726b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#else 727b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.qpcaps = 0x1f; 728b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise#endif 729b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.tcp_emss = qhp->ep->emss; 730b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.ord = qhp->attr.max_ord; 731b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.ird = qhp->attr.max_ird; 732b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.qp_dma_addr = qhp->wq.dma_addr; 733b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.qp_dma_size = (1UL << qhp->wq.size_log2); 734b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.flags = rqes_posted(qhp) ? RECVS_POSTED : 0; 735de3d353072f9342f04112ba0504c3e294220cb8fSteve Wise init_attr.irs = qhp->ep->rcv_seq; 736b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s init_attr.rq_addr 0x%x init_attr.rq_size = %d " 737b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise "flags 0x%x qpcaps 0x%x\n", __FUNCTION__, 738b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.rq_addr, init_attr.rq_size, 739b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise init_attr.flags, init_attr.qpcaps); 740b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = cxio_rdma_init(&rhp->rdev, &init_attr); 741b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s ret %d\n", __FUNCTION__, ret); 742b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return ret; 743b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 744b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 745b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wiseint iwch_modify_qp(struct iwch_dev *rhp, struct iwch_qp *qhp, 746b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise enum iwch_qp_attr_mask mask, 747b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_qp_attributes *attrs, 748b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int internal) 749b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 750b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int ret = 0; 751b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_qp_attributes newattr = qhp->attr; 752b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise unsigned long flag; 753b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int disconnect = 0; 754b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int terminate = 0; 755b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int abort = 0; 756b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int free = 0; 757b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_ep *ep = NULL; 758b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 759b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s qhp %p qpid 0x%x ep %p state %d -> %d\n", __FUNCTION__, 760b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp, qhp->wq.qpid, qhp->ep, qhp->attr.state, 761b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise (mask & IWCH_QP_ATTR_NEXT_STATE) ? attrs->next_state : -1); 762b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 763b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_lock_irqsave(&qhp->lock, flag); 764b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 765b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* Process attr changes if in IDLE */ 766b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (mask & IWCH_QP_ATTR_VALID_MODIFY) { 767b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (qhp->attr.state != IWCH_QP_STATE_IDLE) { 768b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EIO; 769b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 770b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 771b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (mask & IWCH_QP_ATTR_ENABLE_RDMA_READ) 772b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise newattr.enable_rdma_read = attrs->enable_rdma_read; 773b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (mask & IWCH_QP_ATTR_ENABLE_RDMA_WRITE) 774b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise newattr.enable_rdma_write = attrs->enable_rdma_write; 775b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (mask & IWCH_QP_ATTR_ENABLE_RDMA_BIND) 776b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise newattr.enable_bind = attrs->enable_bind; 777b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (mask & IWCH_QP_ATTR_MAX_ORD) { 778b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (attrs->max_ord > 779b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise rhp->attr.max_rdma_read_qp_depth) { 780b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 781b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 782b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 783b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise newattr.max_ord = attrs->max_ord; 784b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 785b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (mask & IWCH_QP_ATTR_MAX_IRD) { 786b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (attrs->max_ird > 787b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise rhp->attr.max_rdma_reads_per_qp) { 788b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 789b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 790b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 791b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise newattr.max_ird = attrs->max_ird; 792b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 793b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr = newattr; 794b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 795b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 796b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!(mask & IWCH_QP_ATTR_NEXT_STATE)) 797b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 798b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (qhp->attr.state == attrs->next_state) 799b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 800b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 801b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise switch (qhp->attr.state) { 802b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_IDLE: 803b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise switch (attrs->next_state) { 804b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_RTS: 805b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!(mask & IWCH_QP_ATTR_LLP_STREAM_HANDLE)) { 806b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 807b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 808b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 809b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!(mask & IWCH_QP_ATTR_MPA_ATTR)) { 810b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 811b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 812b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 813b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.mpa_attr = attrs->mpa_attr; 814b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.llp_stream_handle = attrs->llp_stream_handle; 815b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->ep = qhp->attr.llp_stream_handle; 816b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.state = IWCH_QP_STATE_RTS; 817b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 818b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* 819b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * Ref the endpoint here and deref when we 820b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * disassociate the endpoint from the QP. This 821b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * happens in CLOSING->IDLE transition or *->ERROR 822b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * transition. 823b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise */ 824b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise get_ep(&qhp->ep->com); 825b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 826b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = rdma_init(rhp, qhp, mask, attrs); 827b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_lock_irqsave(&qhp->lock, flag); 828b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (ret) 829b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto err; 830b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 831b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_ERROR: 832b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.state = IWCH_QP_STATE_ERROR; 833b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise flush_qp(qhp, &flag); 834b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 835b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise default: 836b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 837b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 838b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 839b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 840b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_RTS: 841b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise switch (attrs->next_state) { 842b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_CLOSING: 843b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise BUG_ON(atomic_read(&qhp->ep->com.kref.refcount) < 2); 844b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.state = IWCH_QP_STATE_CLOSING; 845b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!internal) { 846b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise abort=0; 847b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise disconnect = 1; 848b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ep = qhp->ep; 849b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 850b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 851b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_TERMINATE: 852b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.state = IWCH_QP_STATE_TERMINATE; 853a1a750523b58cc4bb5a94fbb275a6f2a8bd9ace7Steve Wise if (t3b_device(qhp->rhp)) 854a1a750523b58cc4bb5a94fbb275a6f2a8bd9ace7Steve Wise cxio_set_wq_in_error(&qhp->wq); 855b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!internal) 856b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise terminate = 1; 857b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 858b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_ERROR: 859b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.state = IWCH_QP_STATE_ERROR; 860b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!internal) { 861b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise abort=1; 862b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise disconnect = 1; 863b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ep = qhp->ep; 864b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 865b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto err; 866b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 867b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise default: 868b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 869b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 870b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 871b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 872b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_CLOSING: 873b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!internal) { 874b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 875b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 876b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 877b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise switch (attrs->next_state) { 878b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_IDLE: 879b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.state = IWCH_QP_STATE_IDLE; 880b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.llp_stream_handle = NULL; 881b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise put_ep(&qhp->ep->com); 882b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->ep = NULL; 883b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wake_up(&qhp->wait); 884b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 885b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_ERROR: 886b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto err; 887b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise default: 888b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 889b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto err; 890b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 891b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 892b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_ERROR: 893b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (attrs->next_state != IWCH_QP_STATE_IDLE) { 894b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 895b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 896b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 897b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 898b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!Q_EMPTY(qhp->wq.sq_rptr, qhp->wq.sq_wptr) || 899b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise !Q_EMPTY(qhp->wq.rq_rptr, qhp->wq.rq_wptr)) { 900b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 901b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 902b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 903b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.state = IWCH_QP_STATE_IDLE; 904b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise memset(&qhp->attr, 0, sizeof(qhp->attr)); 905b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 906b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise case IWCH_QP_STATE_TERMINATE: 907b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!internal) { 908b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 909b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 910b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 911b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto err; 912b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 913b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise default: 914b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise printk(KERN_ERR "%s in a bad state %d\n", 915b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise __FUNCTION__, qhp->attr.state); 916b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ret = -EINVAL; 917b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto err; 918b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise break; 919b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 920b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise goto out; 921b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wiseerr: 922b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s disassociating ep %p qpid 0x%x\n", __FUNCTION__, qhp->ep, 923b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->wq.qpid); 924b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 925b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* disassociate the LLP connection */ 926b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.llp_stream_handle = NULL; 927b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise ep = qhp->ep; 928b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->ep = NULL; 929b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->attr.state = IWCH_QP_STATE_ERROR; 930b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise free=1; 931b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise wake_up(&qhp->wait); 932b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise BUG_ON(!ep); 933b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise flush_qp(qhp, &flag); 934b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wiseout: 935b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irqrestore(&qhp->lock, flag); 936b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 937b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (terminate) 938b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise iwch_post_terminate(qhp, NULL); 939b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 940b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* 941b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * If disconnect is 1, then we need to initiate a disconnect 942b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * on the EP. This can be a normal close (RTS->CLOSING) or 943b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * an abnormal close (RTS/CLOSING->ERROR). 944b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise */ 945b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (disconnect) 946b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise iwch_ep_disconnect(ep, abort, GFP_KERNEL); 947b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 948b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise /* 949b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * If free is 1, then we've disassociated the EP from the QP 950b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise * and we need to dereference the EP. 951b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise */ 952b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (free) 953b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise put_ep(&ep->com); 954b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 955b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise PDBG("%s exit state %d\n", __FUNCTION__, qhp->attr.state); 956b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return ret; 957b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 958b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 959b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wisestatic int quiesce_qp(struct iwch_qp *qhp) 960b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 961b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_lock_irq(&qhp->lock); 962b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise iwch_quiesce_tid(qhp->ep); 963b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->flags |= QP_QUIESCED; 964b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irq(&qhp->lock); 965b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return 0; 966b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 967b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 968b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wisestatic int resume_qp(struct iwch_qp *qhp) 969b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 970b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_lock_irq(&qhp->lock); 971b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise iwch_resume_tid(qhp->ep); 972b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp->flags &= ~QP_QUIESCED; 973b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise spin_unlock_irq(&qhp->lock); 974b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return 0; 975b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 976b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 977b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wiseint iwch_quiesce_qps(struct iwch_cq *chp) 978b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 979b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int i; 980b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_qp *qhp; 981b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 982b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise for (i=0; i < T3_MAX_NUM_QP; i++) { 983b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp = get_qhp(chp->rhp, i); 984b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!qhp) 985b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise continue; 986b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if ((qhp->attr.rcq == chp->cq.cqid) && !qp_quiesced(qhp)) { 987b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise quiesce_qp(qhp); 988b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise continue; 989b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 990b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if ((qhp->attr.scq == chp->cq.cqid) && !qp_quiesced(qhp)) 991b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise quiesce_qp(qhp); 992b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 993b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return 0; 994b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 995b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 996b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wiseint iwch_resume_qps(struct iwch_cq *chp) 997b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise{ 998b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise int i; 999b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise struct iwch_qp *qhp; 1000b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise 1001b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise for (i=0; i < T3_MAX_NUM_QP; i++) { 1002b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise qhp = get_qhp(chp->rhp, i); 1003b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if (!qhp) 1004b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise continue; 1005b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if ((qhp->attr.rcq == chp->cq.cqid) && qp_quiesced(qhp)) { 1006b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise resume_qp(qhp); 1007b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise continue; 1008b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 1009b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise if ((qhp->attr.scq == chp->cq.cqid) && qp_quiesced(qhp)) 1010b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise resume_qp(qhp); 1011b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise } 1012b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise return 0; 1013b038ced7b3705bf0ac9b30e118af0f56ab48b847Steve Wise} 1014