1f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/* 2f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Copyright (c) 2006, 2007, 2008, 2009, 2010 QLogic Corporation. 3f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * All rights reserved. 4f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. 5f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 6f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This software is available to you under a choice of one of two 7f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * licenses. You may choose to be licensed under the terms of the GNU 8f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * General Public License (GPL) Version 2, available from the file 9f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * COPYING in the main directory of this source tree, or the 10f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * OpenIB.org BSD license below: 11f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 12f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Redistribution and use in source and binary forms, with or 13f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * without modification, are permitted provided that the following 14f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * conditions are met: 15f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 16f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * - Redistributions of source code must retain the above 17f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * copyright notice, this list of conditions and the following 18f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * disclaimer. 19f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 20f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * - Redistributions in binary form must reproduce the above 21f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * copyright notice, this list of conditions and the following 22f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * disclaimer in the documentation and/or other materials 23f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * provided with the distribution. 24f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 25f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * SOFTWARE. 33f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 34f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 35f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell#include "qib.h" 36f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 37f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/* cut down ridiculously long IB macro names */ 38f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell#define OP(x) IB_OPCODE_UC_##x 39f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 40f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/** 41f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * qib_make_uc_req - construct a request packet (SEND, RDMA write) 42f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @qp: a pointer to the QP 43f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 44f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Return 1 if constructed; otherwise, return 0. 45f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 46f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellint qib_make_uc_req(struct qib_qp *qp) 47f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 48f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_other_headers *ohdr; 49f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_swqe *wqe; 50f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned long flags; 51f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 hwords; 52f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 bth0; 53f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 len; 54cc6ea1385b43487f6ef03bdc91416c8366d28311Mike Marciniszyn u32 pmtu = qp->pmtu; 55f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int ret = 0; 56f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 57f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 58f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 59f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_SEND_OK)) { 60f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(ib_qib_state_ops[qp->state] & QIB_FLUSH_SEND)) 61f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 62f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* We are in the error state, flush the work request. */ 63f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_last == qp->s_head) 64f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 65f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* If DMAs are in progress, we can't flush immediately. */ 66f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (atomic_read(&qp->s_dma_busy)) { 67f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_WAIT_DMA; 68f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 69f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 70f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_last); 71f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); 72f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto done; 73f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 74f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 751c94283ddbe8a9945c4aaac8b0be90d47f97f2dfMike Marciniszyn ohdr = &qp->s_hdr->u.oth; 76f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->remote_ah_attr.ah_flags & IB_AH_GRH) 771c94283ddbe8a9945c4aaac8b0be90d47f97f2dfMike Marciniszyn ohdr = &qp->s_hdr->u.l.oth; 78f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 79f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* header size in 32-bit words LRH+BTH = (8+12)/4. */ 80f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords = 5; 81f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 = 0; 82f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 83f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Get the next send request. */ 84f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_cur); 85f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_wqe = NULL; 86f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (qp->s_state) { 87f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: 88f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(ib_qib_state_ops[qp->state] & 89f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell QIB_PROCESS_NEXT_SEND_OK)) 90f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 91f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check if send work queue is empty. */ 92f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_cur == qp->s_head) 93f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 94f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 95f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Start a new request. 96f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 97f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->psn = qp->s_next_psn; 98f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_psn = qp->s_next_psn; 99f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sge.sge = wqe->sg_list[0]; 100f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sge.sg_list = wqe->sg_list + 1; 101f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sge.num_sge = wqe->wr.num_sge; 102f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sge.total_len = wqe->length; 103f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = wqe->length; 104f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_len = len; 105f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (wqe->wr.opcode) { 106f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_SEND: 107f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_SEND_WITH_IMM: 108f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len > pmtu) { 109f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_FIRST); 110f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = pmtu; 111f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 112f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 113f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_SEND) 114f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_ONLY); 115f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else { 116f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = 117f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell OP(SEND_ONLY_WITH_IMMEDIATE); 118f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Immediate data comes after the BTH */ 119f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.imm_data = wqe->wr.ex.imm_data; 120f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += 1; 121f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 122f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.send_flags & IB_SEND_SOLICITED) 123f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 |= IB_BTH_SOLICITED; 124f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_wqe = wqe; 125f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_cur >= qp->s_size) 126f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = 0; 127f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 128f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 129f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_RDMA_WRITE: 130f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_RDMA_WRITE_WITH_IMM: 131f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.reth.vaddr = 132f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be64(wqe->wr.wr.rdma.remote_addr); 133f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.reth.rkey = 134f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be32(wqe->wr.wr.rdma.rkey); 135f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.reth.length = cpu_to_be32(len); 136f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += sizeof(struct ib_reth) / 4; 137f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len > pmtu) { 138f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_WRITE_FIRST); 139f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = pmtu; 140f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 141f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 142f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_RDMA_WRITE) 143f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_WRITE_ONLY); 144f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else { 145f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = 146f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE); 147f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Immediate data comes after the RETH */ 148f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.imm_data = wqe->wr.ex.imm_data; 149f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += 1; 150f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.send_flags & IB_SEND_SOLICITED) 151f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 |= IB_BTH_SOLICITED; 152f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 153f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_wqe = wqe; 154f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_cur >= qp->s_size) 155f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = 0; 156f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 157f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 158f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: 159f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 160f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 161f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 162f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 163f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_FIRST): 164f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_MIDDLE); 165f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 166f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_MIDDLE): 167f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = qp->s_len; 168f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len > pmtu) { 169f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = pmtu; 170f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 171f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 172f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_SEND) 173f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_LAST); 174f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else { 175f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); 176f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Immediate data comes after the BTH */ 177f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.imm_data = wqe->wr.ex.imm_data; 178f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += 1; 179f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 180f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.send_flags & IB_SEND_SOLICITED) 181f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 |= IB_BTH_SOLICITED; 182f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_wqe = wqe; 183f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_cur >= qp->s_size) 184f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = 0; 185f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 186f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 187f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_FIRST): 188f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_WRITE_MIDDLE); 189f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 190f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_MIDDLE): 191f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = qp->s_len; 192f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len > pmtu) { 193f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = pmtu; 194f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 195f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 196f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_RDMA_WRITE) 197f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_WRITE_LAST); 198f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else { 199f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = 200f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); 201f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Immediate data comes after the BTH */ 202f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.imm_data = wqe->wr.ex.imm_data; 203f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += 1; 204f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.send_flags & IB_SEND_SOLICITED) 205f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 |= IB_BTH_SOLICITED; 206f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 207f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_wqe = wqe; 208f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_cur >= qp->s_size) 209f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = 0; 210f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 211f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 212f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_len -= len; 213f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_hdrwords = hwords; 214f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur_sge = &qp->s_sge; 215f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur_size = len; 216f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), 217f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_next_psn++ & QIB_PSN_MASK); 218f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbelldone: 219f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ret = 1; 220f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto unlock; 221f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 222f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellbail: 223f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~QIB_S_BUSY; 224f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellunlock: 225f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 226f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return ret; 227f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 228f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 229f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/** 230f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * qib_uc_rcv - handle an incoming UC packet 231f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @ibp: the port the packet came in on 232f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @hdr: the header of the packet 233f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @has_grh: true if the packet has a GRH 234f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @data: the packet data 235f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @tlen: the length of the packet 236f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @qp: the QP for this packet. 237f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 238f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This is called from qib_qp_rcv() to process an incoming UC packet 239f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * for the given QP. 240f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Called at interrupt level. 241f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 242f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellvoid qib_uc_rcv(struct qib_ibport *ibp, struct qib_ib_header *hdr, 243f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int has_grh, void *data, u32 tlen, struct qib_qp *qp) 244f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 245f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_other_headers *ohdr; 246f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 opcode; 247f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 hdrsize; 248f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 psn; 249f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 pad; 250f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct ib_wc wc; 251cc6ea1385b43487f6ef03bdc91416c8366d28311Mike Marciniszyn u32 pmtu = qp->pmtu; 252f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct ib_reth *reth; 253f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int ret; 254f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 255f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check for GRH */ 256f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!has_grh) { 257f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr = &hdr->u.oth; 258f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdrsize = 8 + 12; /* LRH + BTH */ 259f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else { 260f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr = &hdr->u.l.oth; 261f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdrsize = 8 + 40 + 12; /* LRH + GRH + BTH */ 262f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 263f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 264f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode = be32_to_cpu(ohdr->bth[0]); 265f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_ruc_check_hdr(ibp, hdr, has_grh, qp, opcode)) 2669fd5473deb421eb7e5575a5f9d7e43ca67c04fe9Mike Marciniszyn return; 267f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 268f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell psn = be32_to_cpu(ohdr->bth[2]); 269f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode >>= 24; 270f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 271f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Compare the PSN verses the expected PSN. */ 272f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(qib_cmp24(psn, qp->r_psn) != 0)) { 273f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 274f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Handle a sequence error. 275f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Silently drop any current message. 276f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 277f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_psn = psn; 278f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellinv: 279f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->r_state == OP(SEND_FIRST) || 280f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_state == OP(SEND_MIDDLE)) { 281f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell set_bit(QIB_R_REWIND_SGE, &qp->r_aflags); 282f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.num_sge = 0; 283f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else 2846a82649f217023863d6b1740017e6c3dd6685327Mike Marciniszyn qib_put_ss(&qp->r_sge); 285f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_state = OP(SEND_LAST); 286f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (opcode) { 287f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_FIRST): 288f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_ONLY): 289f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_ONLY_WITH_IMMEDIATE): 290f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto send_first; 291f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 292f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_FIRST): 293f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_ONLY): 294f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): 295f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto rdma_first; 296f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 297f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: 298f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto drop; 299f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 300f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 301f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 302f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check for opcode sequence errors. */ 303f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (qp->r_state) { 304f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_FIRST): 305f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_MIDDLE): 306f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode == OP(SEND_MIDDLE) || 307f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(SEND_LAST) || 308f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(SEND_LAST_WITH_IMMEDIATE)) 309f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 310f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto inv; 311f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 312f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_FIRST): 313f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_MIDDLE): 314f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode == OP(RDMA_WRITE_MIDDLE) || 315f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(RDMA_WRITE_LAST) || 316f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) 317f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 318f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto inv; 319f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 320f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: 321f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode == OP(SEND_FIRST) || 322f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(SEND_ONLY) || 323f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(SEND_ONLY_WITH_IMMEDIATE) || 324f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(RDMA_WRITE_FIRST) || 325f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(RDMA_WRITE_ONLY) || 326f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE)) 327f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 328f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto inv; 329f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 330f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 331f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->state == IB_QPS_RTR && !(qp->r_flags & QIB_R_COMM_EST)) { 332f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_flags |= QIB_R_COMM_EST; 333f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->ibqp.event_handler) { 334f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct ib_event ev; 335f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 336f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ev.device = qp->ibqp.device; 337f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ev.element.qp = &qp->ibqp; 338f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ev.event = IB_EVENT_COMM_EST; 339f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); 340f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 341f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 342f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 343f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* OK, process the packet. */ 344f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (opcode) { 345f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_FIRST): 346f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_ONLY): 347f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_ONLY_WITH_IMMEDIATE): 348f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellsend_first: 349f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (test_and_clear_bit(QIB_R_REWIND_SGE, &qp->r_aflags)) 350f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge = qp->s_rdma_read_sge; 351f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else { 352f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ret = qib_get_rwqe(qp, 0); 353f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (ret < 0) 354f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto op_err; 355f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!ret) 356f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto drop; 357f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 358f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * qp->s_rdma_read_sge will be the owner 359f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * of the mr references. 360f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 361f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_rdma_read_sge = qp->r_sge; 362f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 363f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_rcv_len = 0; 364f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode == OP(SEND_ONLY)) 3652fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszyn goto no_immediate_data; 366f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else if (opcode == OP(SEND_ONLY_WITH_IMMEDIATE)) 367f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto send_last_imm; 368f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 369f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_MIDDLE): 370f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check for invalid length PMTU or posted rwqe len. */ 371f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(tlen != (hdrsize + pmtu + 4))) 372f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto rewind; 373f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_rcv_len += pmtu; 374f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(qp->r_rcv_len > qp->r_len)) 375f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto rewind; 376f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_copy_sge(&qp->r_sge, data, pmtu, 0); 377f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 378f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 379f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_LAST_WITH_IMMEDIATE): 380f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellsend_last_imm: 381f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.ex.imm_data = ohdr->u.imm_data; 382f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdrsize += 4; 383f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.wc_flags = IB_WC_WITH_IMM; 3842fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszyn goto send_last; 385f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_LAST): 3862fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszynno_immediate_data: 3872fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszyn wc.ex.imm_data = 0; 3882fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszyn wc.wc_flags = 0; 389f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellsend_last: 390f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Get the number of bytes the message was padded by. */ 391f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 392f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check for invalid length. */ 393f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* XXX LAST len should be >= 1 */ 394f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(tlen < (hdrsize + pad + 4))) 395f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto rewind; 396f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Don't count the CRC. */ 397f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell tlen -= (hdrsize + pad + 4); 398f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.byte_len = tlen + qp->r_rcv_len; 399f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(wc.byte_len > qp->r_len)) 400f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto rewind; 401f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.opcode = IB_WC_RECV; 402f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_copy_sge(&qp->r_sge, data, tlen, 0); 4036a82649f217023863d6b1740017e6c3dd6685327Mike Marciniszyn qib_put_ss(&qp->s_rdma_read_sge); 404354dff1bd8ccd41b6e8421226d586d35e7fb8920Mike Marciniszynlast_imm: 405f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.wr_id = qp->r_wr_id; 406f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.status = IB_WC_SUCCESS; 407f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.qp = &qp->ibqp; 408f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.src_qp = qp->remote_qpn; 409f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.slid = qp->remote_ah_attr.dlid; 410f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.sl = qp->remote_ah_attr.sl; 4112fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszyn /* zero fields that are N/A */ 4122fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszyn wc.vendor_err = 0; 4132fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszyn wc.pkey_index = 0; 4142fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszyn wc.dlid_path_bits = 0; 4152fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszyn wc.port_num = 0; 416f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Signal completion event if the solicited bit is set. */ 417f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 418f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (ohdr->bth[0] & 419f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be32(IB_BTH_SOLICITED)) != 0); 420f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 421f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 422f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_FIRST): 423f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_ONLY): 424f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): /* consume RWQE */ 425f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellrdma_first: 426f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(!(qp->qp_access_flags & 427f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell IB_ACCESS_REMOTE_WRITE))) { 428f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto drop; 429f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 430f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell reth = &ohdr->u.rc.reth; 431f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdrsize += sizeof(*reth); 432f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_len = be32_to_cpu(reth->length); 433f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_rcv_len = 0; 434f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.sg_list = NULL; 435f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->r_len != 0) { 436f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 rkey = be32_to_cpu(reth->rkey); 437f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u64 vaddr = be64_to_cpu(reth->vaddr); 438f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int ok; 439f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 440f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check rkey */ 441f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ok = qib_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, 442f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell vaddr, rkey, IB_ACCESS_REMOTE_WRITE); 443f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(!ok)) 444f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto drop; 445f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.num_sge = 1; 446f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else { 447f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.num_sge = 0; 448f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.sge.mr = NULL; 449f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.sge.vaddr = NULL; 450f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.sge.length = 0; 451f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.sge.sge_length = 0; 452f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 453f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode == OP(RDMA_WRITE_ONLY)) 454f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto rdma_last; 4555715f5d44b93507693feca6c62290e123e5ef688Jason Gunthorpe else if (opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE)) { 4565715f5d44b93507693feca6c62290e123e5ef688Jason Gunthorpe wc.ex.imm_data = ohdr->u.rc.imm_data; 457f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto rdma_last_imm; 4585715f5d44b93507693feca6c62290e123e5ef688Jason Gunthorpe } 459f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 460f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_MIDDLE): 461f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check for invalid length PMTU or posted rwqe len. */ 462f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(tlen != (hdrsize + pmtu + 4))) 463f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto drop; 464f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_rcv_len += pmtu; 465f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(qp->r_rcv_len > qp->r_len)) 466f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto drop; 467f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_copy_sge(&qp->r_sge, data, pmtu, 1); 468f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 469f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 470f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE): 471f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.ex.imm_data = ohdr->u.imm_data; 4725715f5d44b93507693feca6c62290e123e5ef688Jason Gunthorperdma_last_imm: 473f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdrsize += 4; 474f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.wc_flags = IB_WC_WITH_IMM; 475f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 476f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Get the number of bytes the message was padded by. */ 477f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 478f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check for invalid length. */ 479f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* XXX LAST len should be >= 1 */ 480f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(tlen < (hdrsize + pad + 4))) 481f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto drop; 482f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Don't count the CRC. */ 483f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell tlen -= (hdrsize + pad + 4); 484f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(tlen + qp->r_rcv_len != qp->r_len)) 485f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto drop; 486f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (test_and_clear_bit(QIB_R_REWIND_SGE, &qp->r_aflags)) 4876a82649f217023863d6b1740017e6c3dd6685327Mike Marciniszyn qib_put_ss(&qp->s_rdma_read_sge); 488f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else { 489f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ret = qib_get_rwqe(qp, 1); 490f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (ret < 0) 491f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto op_err; 492f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!ret) 493f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto drop; 494f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 495f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.byte_len = qp->r_len; 496f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.opcode = IB_WC_RECV_RDMA_WITH_IMM; 497354dff1bd8ccd41b6e8421226d586d35e7fb8920Mike Marciniszyn qib_copy_sge(&qp->r_sge, data, tlen, 1); 4986a82649f217023863d6b1740017e6c3dd6685327Mike Marciniszyn qib_put_ss(&qp->r_sge); 499f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto last_imm; 500f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 501f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_LAST): 502f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellrdma_last: 503f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Get the number of bytes the message was padded by. */ 504f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 505f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check for invalid length. */ 506f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* XXX LAST len should be >= 1 */ 507f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(tlen < (hdrsize + pad + 4))) 508f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto drop; 509f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Don't count the CRC. */ 510f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell tlen -= (hdrsize + pad + 4); 511f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(tlen + qp->r_rcv_len != qp->r_len)) 512f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto drop; 513f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_copy_sge(&qp->r_sge, data, tlen, 1); 5146a82649f217023863d6b1740017e6c3dd6685327Mike Marciniszyn qib_put_ss(&qp->r_sge); 515f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 516f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 517f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: 518f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Drop packet for unknown opcodes. */ 519f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto drop; 520f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 521f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_psn++; 522f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_state = opcode; 523f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return; 524f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 525f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellrewind: 526f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell set_bit(QIB_R_REWIND_SGE, &qp->r_aflags); 527f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.num_sge = 0; 528f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbelldrop: 529f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_pkt_drops++; 530f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return; 531f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 532f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellop_err: 533f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_rc_error(qp, IB_WC_LOC_QP_OP_ERR); 534f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return; 535f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 536f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 537