qib_rc.c revision 414ed90cee32486c50f91b28990443e0dc21c868
1f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/* 2f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Copyright (c) 2006, 2007, 2008, 2009 QLogic Corporation. All rights reserved. 3f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. 4f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 5f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This software is available to you under a choice of one of two 6f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * licenses. You may choose to be licensed under the terms of the GNU 7f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * General Public License (GPL) Version 2, available from the file 8f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * COPYING in the main directory of this source tree, or the 9f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * OpenIB.org BSD license below: 10f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 11f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Redistribution and use in source and binary forms, with or 12f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * without modification, are permitted provided that the following 13f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * conditions are met: 14f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 15f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * - Redistributions of source code must retain the above 16f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * copyright notice, this list of conditions and the following 17f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * disclaimer. 18f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 19f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * - Redistributions in binary form must reproduce the above 20f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * copyright notice, this list of conditions and the following 21f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * disclaimer in the documentation and/or other materials 22f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * provided with the distribution. 23f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 24f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * SOFTWARE. 32f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 33f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 34f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell#include <linux/io.h> 35f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 36f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell#include "qib.h" 37f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 38f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/* cut down ridiculously long IB macro names */ 39f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell#define OP(x) IB_OPCODE_RC_##x 40f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 41f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic void rc_timeout(unsigned long arg); 42f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 43f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic u32 restart_sge(struct qib_sge_state *ss, struct qib_swqe *wqe, 44f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 psn, u32 pmtu) 45f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 46f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 len; 47f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 48f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = ((psn - wqe->psn) & QIB_PSN_MASK) * pmtu; 49f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ss->sge = wqe->sg_list[0]; 50f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ss->sg_list = wqe->sg_list + 1; 51f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ss->num_sge = wqe->wr.num_sge; 52f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ss->total_len = wqe->length; 53f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_skip_sge(ss, len, 0); 54f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return wqe->length - len; 55f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 56f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 57f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic void start_timer(struct qib_qp *qp) 58f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 59f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_TIMER; 60f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_timer.function = rc_timeout; 61f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 4.096 usec. * (1 << qp->timeout) */ 62f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_timer.expires = jiffies + 63f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell usecs_to_jiffies((4096UL * (1UL << qp->timeout)) / 1000UL); 64f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell add_timer(&qp->s_timer); 65f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 66f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 67f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/** 68f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * qib_make_rc_ack - construct a response packet (ACK, NAK, or RDMA read) 69f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @dev: the device for this QP 70f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @qp: a pointer to the QP 71f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @ohdr: a pointer to the IB header being constructed 72f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @pmtu: the path MTU 73f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 74f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Return 1 if constructed; otherwise, return 0. 75f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Note that we are in the responder's side of the QP context. 76f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Note the QP s_lock must be held. 77f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 78f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic int qib_make_rc_ack(struct qib_ibdev *dev, struct qib_qp *qp, 79f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_other_headers *ohdr, u32 pmtu) 80f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 81f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ack_entry *e; 82f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 hwords; 83f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 len; 84f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 bth0; 85f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 bth2; 86f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 87f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Don't send an ACK if we aren't supposed to. */ 88f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) 89f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 90f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 91f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* header size in 32-bit words LRH+BTH = (8+12)/4. */ 92f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords = 5; 93f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 94f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (qp->s_ack_state) { 95f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_LAST): 96f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_ONLY): 97f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e = &qp->s_ack_queue[qp->s_tail_ack_queue]; 98f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (e->rdma_sge.mr) { 99f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_dec(&e->rdma_sge.mr->refcount); 100f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.mr = NULL; 101f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 102f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 103f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(ATOMIC_ACKNOWLEDGE): 104f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 105f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We can increment the tail pointer now that the last 106f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * response has been sent instead of only being 107f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * constructed. 108f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 109f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_tail_ack_queue > QIB_MAX_RDMA_ATOMIC) 110f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_tail_ack_queue = 0; 111f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 112f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_ONLY): 113f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(ACKNOWLEDGE): 114f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check for no next entry in the queue. */ 115f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->r_head_ack_queue == qp->s_tail_ack_queue) { 116f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & QIB_S_ACK_PENDING) 117f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto normal; 118f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 119f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 120f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 121f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e = &qp->s_ack_queue[qp->s_tail_ack_queue]; 122f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (e->opcode == OP(RDMA_READ_REQUEST)) { 123f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 124f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If a RDMA read response is being resent and 125f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * we haven't seen the duplicate request yet, 126f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * then stop sending the remaining responses the 127f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * responder has seen until the requester resends it. 128f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 129f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = e->rdma_sge.sge_length; 130f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len && !e->rdma_sge.mr) { 131f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_tail_ack_queue = qp->r_head_ack_queue; 132f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 133f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 134f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Copy SGE state in case we need to resend */ 135f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_rdma_mr = e->rdma_sge.mr; 136f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_rdma_mr) 137f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_inc(&qp->s_rdma_mr->refcount); 138f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_rdma_sge.sge = e->rdma_sge; 139f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_rdma_sge.num_sge = 1; 140f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur_sge = &qp->s_ack_rdma_sge; 141f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len > pmtu) { 142f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = pmtu; 143f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST); 144f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else { 145f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY); 146f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->sent = 1; 147f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 148f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.aeth = qib_compute_aeth(qp); 149f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords++; 150f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_rdma_psn = e->psn; 151f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK; 152f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else { 153f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* COMPARE_SWAP or FETCH_ADD */ 154f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur_sge = NULL; 155f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = 0; 156f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); 157f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.at.aeth = qib_compute_aeth(qp); 158f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.at.atomic_ack_eth[0] = 159f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be32(e->atomic_data >> 32); 160f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.at.atomic_ack_eth[1] = 161f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be32(e->atomic_data); 162f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += sizeof(ohdr->u.at) / sizeof(u32); 163f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 = e->psn & QIB_PSN_MASK; 164f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->sent = 1; 165f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 166f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 = qp->s_ack_state << 24; 167f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 168f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 169f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_FIRST): 170f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE); 171f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 172f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_MIDDLE): 173f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur_sge = &qp->s_ack_rdma_sge; 174f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_rdma_mr = qp->s_ack_rdma_sge.sge.mr; 175f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_rdma_mr) 176f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_inc(&qp->s_rdma_mr->refcount); 177f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = qp->s_ack_rdma_sge.sge.sge_length; 178f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len > pmtu) 179f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = pmtu; 180f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else { 181f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.aeth = qib_compute_aeth(qp); 182f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords++; 183f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST); 184f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e = &qp->s_ack_queue[qp->s_tail_ack_queue]; 185f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->sent = 1; 186f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 187f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 = qp->s_ack_state << 24; 188f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK; 189f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 190f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 191f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: 192f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellnormal: 193f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 194f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Send a regular ACK. 195f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Set the s_ack_state so we wait until after sending 196f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * the ACK before setting s_ack_state to ACKNOWLEDGE 197f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * (see above). 198f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 199f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_state = OP(SEND_ONLY); 200f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~QIB_S_ACK_PENDING; 201f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur_sge = NULL; 202f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_nak_state) 203f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.aeth = 204f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be32((qp->r_msn & QIB_MSN_MASK) | 205f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (qp->s_nak_state << 206f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell QIB_AETH_CREDIT_SHIFT)); 207f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else 208f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.aeth = qib_compute_aeth(qp); 209f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords++; 210f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = 0; 211f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 = OP(ACKNOWLEDGE) << 24; 212f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 = qp->s_ack_psn & QIB_PSN_MASK; 213f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 214f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_rdma_ack_cnt++; 215f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_hdrwords = hwords; 216f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur_size = len; 217f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_make_ruc_header(qp, ohdr, bth0, bth2); 218f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return 1; 219f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 220f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellbail: 221f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_state = OP(ACKNOWLEDGE); 222f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~(QIB_S_RESP_PENDING | QIB_S_ACK_PENDING); 223f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return 0; 224f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 225f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 226f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/** 227f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * qib_make_rc_req - construct a request packet (SEND, RDMA r/w, ATOMIC) 228f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @qp: a pointer to the QP 229f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 230f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Return 1 if constructed; otherwise, return 0. 231f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 232f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellint qib_make_rc_req(struct qib_qp *qp) 233f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 234f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ibdev *dev = to_idev(qp->ibqp.device); 235f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_other_headers *ohdr; 236f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_sge_state *ss; 237f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_swqe *wqe; 238f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 hwords; 239f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 len; 240f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 bth0; 241f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 bth2; 242f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu); 243f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell char newreq; 244f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned long flags; 245f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int ret = 0; 246f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int delta; 247f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 248f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr = &qp->s_hdr.u.oth; 249f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->remote_ah_attr.ah_flags & IB_AH_GRH) 250f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr = &qp->s_hdr.u.l.oth; 251f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 252f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 253f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * The lock is needed to synchronize between the sending tasklet, 254f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * the receive interrupt handler, and timeout resends. 255f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 256f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 257f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 258f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Sending responses has higher priority over sending requests. */ 259f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if ((qp->s_flags & QIB_S_RESP_PENDING) && 260f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_make_rc_ack(dev, qp, ohdr, pmtu)) 261f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto done; 262f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 263f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_SEND_OK)) { 264f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(ib_qib_state_ops[qp->state] & QIB_FLUSH_SEND)) 265f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 266f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* We are in the error state, flush the work request. */ 267f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_last == qp->s_head) 268f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 269f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* If DMAs are in progress, we can't flush immediately. */ 270f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (atomic_read(&qp->s_dma_busy)) { 271f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_WAIT_DMA; 272f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 273f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 274f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_last); 275f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell while (qp->s_last != qp->s_acked) { 276f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_send_complete(qp, wqe, IB_WC_SUCCESS); 277f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_last >= qp->s_size) 278f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_last = 0; 279f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_last); 280f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 281f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); 282f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto done; 283f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 284f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 285f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & (QIB_S_WAIT_RNR | QIB_S_WAIT_ACK)) 286f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 287f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 288f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) { 289f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) { 290f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_WAIT_PSN; 291f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 292f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 293f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sending_psn = qp->s_psn; 294f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sending_hpsn = qp->s_psn - 1; 295f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 296f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 297f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* header size in 32-bit words LRH+BTH = (8+12)/4. */ 298f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords = 5; 299f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 = 0; 300f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 301f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Send a request. */ 302f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_cur); 303f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (qp->s_state) { 304f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: 305f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_NEXT_SEND_OK)) 306f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 307f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 308f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Resend an old request or start a new one. 309f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 310f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We keep track of the current SWQE so that 311f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * we don't reset the "furthest progress" state 312f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * if we need to back up. 313f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 314f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell newreq = 0; 315f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_cur == qp->s_tail) { 316f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check if send work queue is empty. */ 317f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_tail == qp->s_head) 318f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 319f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 320f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If a fence is requested, wait for previous 321f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * RDMA read and atomic operations to finish. 322f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 323f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if ((wqe->wr.send_flags & IB_SEND_FENCE) && 324f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_num_rd_atomic) { 325f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_WAIT_FENCE; 326f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 327f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 328f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->psn = qp->s_next_psn; 329f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell newreq = 1; 330f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 331f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 332f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Note that we have to be careful not to modify the 333f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * original work request since we may need to resend 334f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * it. 335f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 336f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = wqe->length; 337f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ss = &qp->s_sge; 338f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 = qp->s_psn & QIB_PSN_MASK; 339f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (wqe->wr.opcode) { 340f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_SEND: 341f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_SEND_WITH_IMM: 342f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* If no credit, return. */ 343f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(qp->s_flags & QIB_S_UNLIMITED_CREDIT) && 344f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) { 345f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_WAIT_SSN_CREDIT; 346f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 347f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 348f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->lpsn = wqe->psn; 349f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len > pmtu) { 350f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->lpsn += (len - 1) / pmtu; 351f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_FIRST); 352f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = pmtu; 353f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 354f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 355f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_SEND) 356f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_ONLY); 357f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else { 358f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE); 359f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Immediate data comes after the BTH */ 360f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.imm_data = wqe->wr.ex.imm_data; 361f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += 1; 362f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 363f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.send_flags & IB_SEND_SOLICITED) 364f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 |= IB_BTH_SOLICITED; 365f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 |= IB_BTH_REQ_ACK; 366f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_cur == qp->s_size) 367f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = 0; 368f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 369f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 370f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_RDMA_WRITE: 371f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (newreq && !(qp->s_flags & QIB_S_UNLIMITED_CREDIT)) 372f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_lsn++; 373f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 374f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_RDMA_WRITE_WITH_IMM: 375f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* If no credit, return. */ 376f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(qp->s_flags & QIB_S_UNLIMITED_CREDIT) && 377f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) { 378f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_WAIT_SSN_CREDIT; 379f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 380f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 381f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.reth.vaddr = 382f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be64(wqe->wr.wr.rdma.remote_addr); 383f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.reth.rkey = 384f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be32(wqe->wr.wr.rdma.rkey); 385f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.reth.length = cpu_to_be32(len); 386f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += sizeof(struct ib_reth) / sizeof(u32); 387f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->lpsn = wqe->psn; 388f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len > pmtu) { 389f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->lpsn += (len - 1) / pmtu; 390f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_WRITE_FIRST); 391f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = pmtu; 392f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 393f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 394f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_RDMA_WRITE) 395f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_WRITE_ONLY); 396f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else { 397f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = 398f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE); 399f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Immediate data comes after RETH */ 400f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.imm_data = wqe->wr.ex.imm_data; 401f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += 1; 402f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.send_flags & IB_SEND_SOLICITED) 403f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 |= IB_BTH_SOLICITED; 404f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 405f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 |= IB_BTH_REQ_ACK; 406f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_cur == qp->s_size) 407f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = 0; 408f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 409f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 410f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_RDMA_READ: 411f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 412f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Don't allow more operations to be started 413f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * than the QP limits allow. 414f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 415f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (newreq) { 416f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_num_rd_atomic >= 417f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_max_rd_atomic) { 418f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_WAIT_RDMAR; 419f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 420f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 421f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_num_rd_atomic++; 422f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(qp->s_flags & QIB_S_UNLIMITED_CREDIT)) 423f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_lsn++; 424f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 425f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Adjust s_next_psn to count the 426f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * expected number of responses. 427f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 428f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len > pmtu) 429f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_next_psn += (len - 1) / pmtu; 430f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->lpsn = qp->s_next_psn++; 431f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 432f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.reth.vaddr = 433f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be64(wqe->wr.wr.rdma.remote_addr); 434f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.reth.rkey = 435f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be32(wqe->wr.wr.rdma.rkey); 436f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.reth.length = cpu_to_be32(len); 437f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_READ_REQUEST); 438f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32); 439f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ss = NULL; 440f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = 0; 441f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 |= IB_BTH_REQ_ACK; 442f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_cur == qp->s_size) 443f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = 0; 444f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 445f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 446f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_ATOMIC_CMP_AND_SWP: 447f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_ATOMIC_FETCH_AND_ADD: 448f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 449f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Don't allow more operations to be started 450f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * than the QP limits allow. 451f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 452f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (newreq) { 453f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_num_rd_atomic >= 454f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_max_rd_atomic) { 455f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_WAIT_RDMAR; 456f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 457f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 458f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_num_rd_atomic++; 459f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(qp->s_flags & QIB_S_UNLIMITED_CREDIT)) 460f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_lsn++; 461f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->lpsn = wqe->psn; 462f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 463f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) { 464f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(COMPARE_SWAP); 465f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.atomic_eth.swap_data = cpu_to_be64( 466f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.wr.atomic.swap); 467f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.atomic_eth.compare_data = cpu_to_be64( 468f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.wr.atomic.compare_add); 469f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else { 470f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(FETCH_ADD); 471f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.atomic_eth.swap_data = cpu_to_be64( 472f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.wr.atomic.compare_add); 473f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.atomic_eth.compare_data = 0; 474f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 475f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.atomic_eth.vaddr[0] = cpu_to_be32( 476f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.wr.atomic.remote_addr >> 32); 477f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.atomic_eth.vaddr[1] = cpu_to_be32( 478f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.wr.atomic.remote_addr); 479f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.atomic_eth.rkey = cpu_to_be32( 480f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.wr.atomic.rkey); 481f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += sizeof(struct ib_atomic_eth) / sizeof(u32); 482f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ss = NULL; 483f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = 0; 484f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 |= IB_BTH_REQ_ACK; 485f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_cur == qp->s_size) 486f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = 0; 487f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 488f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 489f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: 490f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 491f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 492f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sge.sge = wqe->sg_list[0]; 493f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sge.sg_list = wqe->sg_list + 1; 494f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sge.num_sge = wqe->wr.num_sge; 495f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sge.total_len = wqe->length; 496f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_len = wqe->length; 497f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (newreq) { 498f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_tail++; 499f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_tail >= qp->s_size) 500f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_tail = 0; 501f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 502f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_RDMA_READ) 503f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_psn = wqe->lpsn + 1; 504f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else { 505f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_psn++; 506f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(qp->s_psn, qp->s_next_psn) > 0) 507f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_next_psn = qp->s_psn; 508f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 509f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 510f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 511f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_FIRST): 512f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 513f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * qp->s_state is normally set to the opcode of the 514f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * last packet constructed for new requests and therefore 515f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * is never set to RDMA read response. 516f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * RDMA_READ_RESPONSE_FIRST is used by the ACK processing 517f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * thread to indicate a SEND needs to be restarted from an 518f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * earlier PSN without interferring with the sending thread. 519f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * See qib_restart_rc(). 520f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 521f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); 522f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 523f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_FIRST): 524f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_MIDDLE); 525f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 526f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_MIDDLE): 527f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 = qp->s_psn++ & QIB_PSN_MASK; 528f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(qp->s_psn, qp->s_next_psn) > 0) 529f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_next_psn = qp->s_psn; 530f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ss = &qp->s_sge; 531f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = qp->s_len; 532f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len > pmtu) { 533f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = pmtu; 534f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 535f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 536f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_SEND) 537f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_LAST); 538f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else { 539f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); 540f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Immediate data comes after the BTH */ 541f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.imm_data = wqe->wr.ex.imm_data; 542f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += 1; 543f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 544f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.send_flags & IB_SEND_SOLICITED) 545f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 |= IB_BTH_SOLICITED; 546f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 |= IB_BTH_REQ_ACK; 547f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur++; 548f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_cur >= qp->s_size) 549f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = 0; 550f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 551f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 552f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_LAST): 553f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 554f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * qp->s_state is normally set to the opcode of the 555f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * last packet constructed for new requests and therefore 556f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * is never set to RDMA read response. 557f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * RDMA_READ_RESPONSE_LAST is used by the ACK processing 558f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * thread to indicate a RDMA write needs to be restarted from 559f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * an earlier PSN without interferring with the sending thread. 560f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * See qib_restart_rc(). 561f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 562f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); 563f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 564f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_FIRST): 565f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_WRITE_MIDDLE); 566f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 567f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_MIDDLE): 568f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 = qp->s_psn++ & QIB_PSN_MASK; 569f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(qp->s_psn, qp->s_next_psn) > 0) 570f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_next_psn = qp->s_psn; 571f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ss = &qp->s_sge; 572f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = qp->s_len; 573f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len > pmtu) { 574f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = pmtu; 575f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 576f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 577f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_RDMA_WRITE) 578f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_WRITE_LAST); 579f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else { 580f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); 581f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Immediate data comes after the BTH */ 582f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.imm_data = wqe->wr.ex.imm_data; 583f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += 1; 584f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.send_flags & IB_SEND_SOLICITED) 585f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 |= IB_BTH_SOLICITED; 586f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 587f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 |= IB_BTH_REQ_ACK; 588f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur++; 589f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_cur >= qp->s_size) 590f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = 0; 591f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 592f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 593f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_MIDDLE): 594f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 595f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * qp->s_state is normally set to the opcode of the 596f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * last packet constructed for new requests and therefore 597f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * is never set to RDMA read response. 598f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * RDMA_READ_RESPONSE_MIDDLE is used by the ACK processing 599f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * thread to indicate a RDMA read needs to be restarted from 600f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * an earlier PSN without interferring with the sending thread. 601f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * See qib_restart_rc(). 602f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 603f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = ((qp->s_psn - wqe->psn) & QIB_PSN_MASK) * pmtu; 604f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.reth.vaddr = 605f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be64(wqe->wr.wr.rdma.remote_addr + len); 606f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.reth.rkey = 607f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be32(wqe->wr.wr.rdma.rkey); 608f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.reth.length = cpu_to_be32(wqe->length - len); 609f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_READ_REQUEST); 610f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32); 611f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 = (qp->s_psn & QIB_PSN_MASK) | IB_BTH_REQ_ACK; 612f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_psn = wqe->lpsn + 1; 613f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ss = NULL; 614f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = 0; 615f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur++; 616f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_cur == qp->s_size) 617f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = 0; 618f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 619f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 620f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sending_hpsn = bth2; 621f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell delta = (((int) bth2 - (int) wqe->psn) << 8) >> 8; 622f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (delta && delta % QIB_PSN_CREDIT == 0) 623f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 |= IB_BTH_REQ_ACK; 624f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & QIB_S_SEND_ONE) { 625f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~QIB_S_SEND_ONE; 626f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_WAIT_ACK; 627f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 |= IB_BTH_REQ_ACK; 628f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 629f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_len -= len; 630f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_hdrwords = hwords; 631f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur_sge = ss; 632f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur_size = len; 633f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), bth2); 634f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbelldone: 635f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ret = 1; 636f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto unlock; 637f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 638f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellbail: 639f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~QIB_S_BUSY; 640f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellunlock: 641f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 642f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return ret; 643f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 644f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 645f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/** 646f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * qib_send_rc_ack - Construct an ACK packet and send it 647f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @qp: a pointer to the QP 648f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 649f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This is called from qib_rc_rcv() and qib_kreceive(). 650f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Note that RDMA reads and atomics are handled in the 651f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * send side QP state and tasklet. 652f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 653f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellvoid qib_send_rc_ack(struct qib_qp *qp) 654f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 655f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device); 656f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); 657f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_pportdata *ppd = ppd_from_ibp(ibp); 658f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u64 pbc; 659f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u16 lrh0; 660f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 bth0; 661f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 hwords; 662f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 pbufn; 663f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 __iomem *piobuf; 664f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ib_header hdr; 665f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_other_headers *ohdr; 666f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 control; 667f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned long flags; 668f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 669f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 670f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 671f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) 672f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto unlock; 673f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 674f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Don't send ACK or NAK if a RDMA read or atomic is pending. */ 675f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if ((qp->s_flags & QIB_S_RESP_PENDING) || qp->s_rdma_ack_cnt) 676f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto queue_ack; 677f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 678f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Construct the header with s_lock held so APM doesn't change it. */ 679f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr = &hdr.u.oth; 680f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell lrh0 = QIB_LRH_BTH; 681f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* header size in 32-bit words LRH+BTH+AETH = (8+12+4)/4. */ 682f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords = 6; 683f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) { 684f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += qib_make_grh(ibp, &hdr.u.l.grh, 685f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell &qp->remote_ah_attr.grh, hwords, 0); 686f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr = &hdr.u.l.oth; 687f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell lrh0 = QIB_LRH_GRH; 688f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 689f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* read pkey_index w/o lock (its atomic) */ 690f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 = qib_get_pkey(ibp, qp->s_pkey_index) | (OP(ACKNOWLEDGE) << 24); 691f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_mig_state == IB_MIG_MIGRATED) 692f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 |= IB_BTH_MIG_REQ; 693f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->r_nak_state) 694f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.aeth = cpu_to_be32((qp->r_msn & QIB_MSN_MASK) | 695f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (qp->r_nak_state << 696f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell QIB_AETH_CREDIT_SHIFT)); 697f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else 698f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.aeth = qib_compute_aeth(qp); 699f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell lrh0 |= ibp->sl_to_vl[qp->remote_ah_attr.sl] << 12 | 700f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->remote_ah_attr.sl << 4; 701f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdr.lrh[0] = cpu_to_be16(lrh0); 702f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid); 703f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdr.lrh[2] = cpu_to_be16(hwords + SIZE_OF_CRC); 704f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdr.lrh[3] = cpu_to_be16(ppd->lid | qp->remote_ah_attr.src_path_bits); 705f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->bth[0] = cpu_to_be32(bth0); 706f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->bth[1] = cpu_to_be32(qp->remote_qpn); 707f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->bth[2] = cpu_to_be32(qp->r_ack_psn & QIB_PSN_MASK); 708f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 709f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 710f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 711f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Don't try to send ACKs if the link isn't ACTIVE */ 712f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(ppd->lflags & QIBL_LINKACTIVE)) 713f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto done; 714f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 715f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell control = dd->f_setpbc_control(ppd, hwords + SIZE_OF_CRC, 716f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_srate, lrh0 >> 12); 717f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* length is + 1 for the control dword */ 718f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell pbc = ((u64) control << 32) | (hwords + 1); 719f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 720f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell piobuf = dd->f_getsendbuf(ppd, pbc, &pbufn); 721f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!piobuf) { 722f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 723f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We are out of PIO buffers at the moment. 724f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Pass responsibility for sending the ACK to the 725f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * send tasklet so that when a PIO buffer becomes 726f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * available, the ACK is sent ahead of other outgoing 727f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * packets. 728f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 729f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 730f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto queue_ack; 731f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 732f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 733f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 734f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Write the pbc. 735f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We have to flush after the PBC for correctness 736f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * on some cpus or WC buffer can be written out of order. 737f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 738f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell writeq(pbc, piobuf); 739f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 740f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (dd->flags & QIB_PIO_FLUSH_WC) { 741f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 *hdrp = (u32 *) &hdr; 742f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 743f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_flush_wc(); 744f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_pio_copy(piobuf + 2, hdrp, hwords - 1); 745f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_flush_wc(); 746f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell __raw_writel(hdrp[hwords - 1], piobuf + hwords + 1); 747f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else 748f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_pio_copy(piobuf + 2, (u32 *) &hdr, hwords); 749f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 750f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (dd->flags & QIB_USE_SPCL_TRIG) { 751f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 spcl_off = (pbufn >= dd->piobcnt2k) ? 2047 : 1023; 752f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 753f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_flush_wc(); 754f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell __raw_writel(0xaebecede, piobuf + spcl_off); 755f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 756f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 757f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_flush_wc(); 758f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_sendbuf_done(dd, pbufn); 759f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 760f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_unicast_xmit++; 761f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto done; 762f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 763f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellqueue_ack: 764f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK) { 765f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rc_qacks++; 766f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_ACK_PENDING | QIB_S_RESP_PENDING; 767f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_nak_state = qp->r_nak_state; 768f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_psn = qp->r_ack_psn; 769f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 770f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Schedule the send tasklet. */ 771f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 772f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 773f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellunlock: 774f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 775f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbelldone: 776f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return; 777f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 778f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 779f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/** 780f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * reset_psn - reset the QP state to send starting from PSN 781f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @qp: the QP 782f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @psn: the packet sequence number to restart at 783f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 784f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This is called from qib_rc_rcv() to process an incoming RC ACK 785f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * for the given QP. 786f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Called at interrupt level with the QP s_lock held. 787f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 788f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic void reset_psn(struct qib_qp *qp, u32 psn) 789f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 790f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 n = qp->s_acked; 791f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_swqe *wqe = get_swqe_ptr(qp, n); 792f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 opcode; 793f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 794f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = n; 795f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 796f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 797f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If we are starting the request from the beginning, 798f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * let the normal send code handle initialization. 799f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 800f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(psn, wqe->psn) <= 0) { 801f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_LAST); 802f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto done; 803f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 804f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 805f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Find the work request opcode corresponding to the given PSN. */ 806f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode = wqe->wr.opcode; 807f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell for (;;) { 808f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int diff; 809f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 810f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++n == qp->s_size) 811f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell n = 0; 812f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (n == qp->s_tail) 813f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 814f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, n); 815f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell diff = qib_cmp24(psn, wqe->psn); 816f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (diff < 0) 817f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 818f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = n; 819f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 820f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If we are starting the request from the beginning, 821f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * let the normal send code handle initialization. 822f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 823f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (diff == 0) { 824f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_LAST); 825f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto done; 826f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 827f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode = wqe->wr.opcode; 828f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 829f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 830f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 831f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Set the state to restart in the middle of a request. 832f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Don't change the s_sge, s_cur_sge, or s_cur_size. 833f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * See qib_make_rc_req(). 834f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 835f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (opcode) { 836f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_SEND: 837f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_SEND_WITH_IMM: 838f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_READ_RESPONSE_FIRST); 839f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 840f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 841f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_RDMA_WRITE: 842f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_RDMA_WRITE_WITH_IMM: 843f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_READ_RESPONSE_LAST); 844f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 845f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 846f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_RDMA_READ: 847f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE); 848f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 849f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 850f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: 851f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 852f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This case shouldn't happen since its only 853f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * one PSN per req. 854f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 855f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_LAST); 856f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 857f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbelldone: 858f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_psn = psn; 859f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 860f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Set QIB_S_WAIT_PSN as qib_rc_complete() may start the timer 861f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * asynchronously before the send tasklet can get scheduled. 862f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Doing it in qib_make_rc_req() is too late. 863f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 864f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if ((qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) && 865f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) 866f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_WAIT_PSN; 867f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 868f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 869f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/* 870f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Back up requester to resend the last un-ACKed request. 871a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell * The QP r_lock and s_lock should be held and interrupts disabled. 872f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 873f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic void qib_restart_rc(struct qib_qp *qp, u32 psn, int wait) 874f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 875f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_swqe *wqe = get_swqe_ptr(qp, qp->s_acked); 876f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ibport *ibp; 877f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 878f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_retry == 0) { 879f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_mig_state == IB_MIG_ARMED) { 880f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_migrate_qp(qp); 881f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_retry = qp->s_retry_cnt; 882f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else if (qp->s_last == qp->s_acked) { 883f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR); 884f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_error_qp(qp, IB_WC_WR_FLUSH_ERR); 885f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return; 886f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else /* XXX need to handle delayed completion */ 887f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return; 888f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else 889f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_retry--; 890f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 891f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp = to_iport(qp->ibqp.device, qp->port_num); 892f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_RDMA_READ) 893f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rc_resends++; 894f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else 895f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK; 896f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 897f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~(QIB_S_WAIT_FENCE | QIB_S_WAIT_RDMAR | 898f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell QIB_S_WAIT_SSN_CREDIT | QIB_S_WAIT_PSN | 899f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell QIB_S_WAIT_ACK); 900f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wait) 901f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_SEND_ONE; 902f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell reset_psn(qp, psn); 903f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 904f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 905f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/* 906f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This is called from s_timer for missing responses. 907f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 908f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic void rc_timeout(unsigned long arg) 909f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 910f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_qp *qp = (struct qib_qp *)arg; 911f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ibport *ibp; 912f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned long flags; 913f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 914a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell spin_lock_irqsave(&qp->r_lock, flags); 915a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell spin_lock(&qp->s_lock); 916f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & QIB_S_TIMER) { 917f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp = to_iport(qp->ibqp.device, qp->port_num); 918f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rc_timeouts++; 919f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~QIB_S_TIMER; 920f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell del_timer(&qp->s_timer); 921f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_restart_rc(qp, qp->s_last_psn + 1, 1); 922f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 923f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 924a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell spin_unlock(&qp->s_lock); 925a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell spin_unlock_irqrestore(&qp->r_lock, flags); 926f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 927f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 928f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/* 929f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This is called from s_timer for RNR timeouts. 930f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 931f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellvoid qib_rc_rnr_retry(unsigned long arg) 932f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 933f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_qp *qp = (struct qib_qp *)arg; 934f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned long flags; 935f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 936f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 937f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & QIB_S_WAIT_RNR) { 938f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~QIB_S_WAIT_RNR; 939f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell del_timer(&qp->s_timer); 940f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 941f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 942f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 943f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 944f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 945f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/* 946f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Set qp->s_sending_psn to the next PSN after the given one. 947f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This would be psn+1 except when RDMA reads are present. 948f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 949f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic void reset_sending_psn(struct qib_qp *qp, u32 psn) 950f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 951f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_swqe *wqe; 952f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 n = qp->s_last; 953f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 954f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Find the work request corresponding to the given PSN. */ 955f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell for (;;) { 956f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, n); 957f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(psn, wqe->lpsn) <= 0) { 958f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_RDMA_READ) 959f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sending_psn = wqe->lpsn + 1; 960f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else 961f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sending_psn = psn + 1; 962f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 963f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 964f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++n == qp->s_size) 965f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell n = 0; 966f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (n == qp->s_tail) 967f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 968f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 969f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 970f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 971f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/* 972f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This should be called with the QP s_lock held and interrupts disabled. 973f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 974f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellvoid qib_rc_send_complete(struct qib_qp *qp, struct qib_ib_header *hdr) 975f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 976f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_other_headers *ohdr; 977f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_swqe *wqe; 978f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct ib_wc wc; 979f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned i; 980f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 opcode; 981f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 psn; 982f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 983f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_OR_FLUSH_SEND)) 984f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return; 985f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 986f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Find out where the BTH is */ 987f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if ((be16_to_cpu(hdr->lrh[0]) & 3) == QIB_LRH_BTH) 988f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr = &hdr->u.oth; 989f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else 990f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr = &hdr->u.l.oth; 991f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 992f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode = be32_to_cpu(ohdr->bth[0]) >> 24; 993f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) && 994f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode <= OP(ATOMIC_ACKNOWLEDGE)) { 995f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell WARN_ON(!qp->s_rdma_ack_cnt); 996f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_rdma_ack_cnt--; 997f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return; 998f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 999f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1000f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell psn = be32_to_cpu(ohdr->bth[2]); 1001f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell reset_sending_psn(qp, psn); 1002f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1003f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1004f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Start timer after a packet requesting an ACK has been sent and 1005f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * there are still requests that haven't been acked. 1006f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1007f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if ((psn & IB_BTH_REQ_ACK) && qp->s_acked != qp->s_tail && 1008f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell !(qp->s_flags & (QIB_S_TIMER | QIB_S_WAIT_RNR | QIB_S_WAIT_PSN))) 1009f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell start_timer(qp); 1010f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1011f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell while (qp->s_last != qp->s_acked) { 1012f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_last); 1013f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 && 1014f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) 1015f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1016f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell for (i = 0; i < wqe->wr.num_sge; i++) { 1017f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_sge *sge = &wqe->sg_list[i]; 1018f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1019f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_dec(&sge->mr->refcount); 1020f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1021f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Post a send completion queue entry if requested. */ 1022f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(qp->s_flags & QIB_S_SIGNAL_REQ_WR) || 1023f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (wqe->wr.send_flags & IB_SEND_SIGNALED)) { 1024f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell memset(&wc, 0, sizeof wc); 1025f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.wr_id = wqe->wr.wr_id; 1026f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.status = IB_WC_SUCCESS; 1027f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.opcode = ib_qib_wc_opcode[wqe->wr.opcode]; 1028f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.byte_len = wqe->length; 1029f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.qp = &qp->ibqp; 1030f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 0); 1031f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1032f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_last >= qp->s_size) 1033f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_last = 0; 1034f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1035f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1036f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If we were waiting for sends to complete before resending, 1037f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * and they are now complete, restart sending. 1038f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1039f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & QIB_S_WAIT_PSN && 1040f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { 1041f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~QIB_S_WAIT_PSN; 1042f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sending_psn = qp->s_psn; 1043f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sending_hpsn = qp->s_psn - 1; 1044f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 1045f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1046f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 1047f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1048f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic inline void update_last_psn(struct qib_qp *qp, u32 psn) 1049f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 1050f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_last_psn = psn; 1051f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 1052f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1053f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/* 1054f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Generate a SWQE completion. 1055f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This is similar to qib_send_complete but has to check to be sure 1056f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * that the SGEs are not being referenced if the SWQE is being resent. 1057f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1058f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic struct qib_swqe *do_rc_completion(struct qib_qp *qp, 1059f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_swqe *wqe, 1060f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ibport *ibp) 1061f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 1062f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct ib_wc wc; 1063f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned i; 1064f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1065f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1066f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Don't decrement refcount and don't generate a 1067f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * completion if the SWQE is being resent until the send 1068f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * is finished. 1069f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1070f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 || 1071f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { 1072f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell for (i = 0; i < wqe->wr.num_sge; i++) { 1073f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_sge *sge = &wqe->sg_list[i]; 1074f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1075f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_dec(&sge->mr->refcount); 1076f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1077f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Post a send completion queue entry if requested. */ 1078f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(qp->s_flags & QIB_S_SIGNAL_REQ_WR) || 1079f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (wqe->wr.send_flags & IB_SEND_SIGNALED)) { 1080f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell memset(&wc, 0, sizeof wc); 1081f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.wr_id = wqe->wr.wr_id; 1082f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.status = IB_WC_SUCCESS; 1083f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.opcode = ib_qib_wc_opcode[wqe->wr.opcode]; 1084f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.byte_len = wqe->length; 1085f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.qp = &qp->ibqp; 1086f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 0); 1087f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1088f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_last >= qp->s_size) 1089f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_last = 0; 1090f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else 1091f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rc_delayed_comp++; 1092f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1093f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_retry = qp->s_retry_cnt; 1094f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell update_last_psn(qp, wqe->lpsn); 1095f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1096f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1097f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If we are completing a request which is in the process of 1098f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * being resent, we can stop resending it since we know the 1099f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * responder has already seen it. 1100f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1101f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_acked == qp->s_cur) { 1102f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_cur >= qp->s_size) 1103f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = 0; 1104f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_acked = qp->s_cur; 1105f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_cur); 1106f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_acked != qp->s_tail) { 1107f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_LAST); 1108f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_psn = wqe->psn; 1109f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1110f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else { 1111f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_acked >= qp->s_size) 1112f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_acked = 0; 1113f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->state == IB_QPS_SQD && qp->s_acked == qp->s_cur) 1114f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_draining = 0; 1115f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_acked); 1116f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1117f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return wqe; 1118f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 1119f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1120f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/** 1121f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * do_rc_ack - process an incoming RC ACK 1122f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @qp: the QP the ACK came in on 1123f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @psn: the packet sequence number of the ACK 1124f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @opcode: the opcode of the request that resulted in the ACK 1125f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 1126f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This is called from qib_rc_rcv_resp() to process an incoming RC ACK 1127f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * for the given QP. 1128f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Called at interrupt level with the QP s_lock held. 1129f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Returns 1 if OK, 0 if current operation should be aborted (NAK). 1130f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1131f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic int do_rc_ack(struct qib_qp *qp, u32 aeth, u32 psn, int opcode, 1132f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u64 val, struct qib_ctxtdata *rcd) 1133f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 1134f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ibport *ibp; 1135f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell enum ib_wc_status status; 1136f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_swqe *wqe; 1137f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int ret = 0; 1138f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 ack_psn; 1139f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int diff; 1140f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1141f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Remove QP from retry timer */ 1142f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & (QIB_S_TIMER | QIB_S_WAIT_RNR)) { 1143f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~(QIB_S_TIMER | QIB_S_WAIT_RNR); 1144f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell del_timer(&qp->s_timer); 1145f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1146f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1147f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1148f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Note that NAKs implicitly ACK outstanding SEND and RDMA write 1149f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * requests and implicitly NAK RDMA read and atomic requests issued 1150f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * before the NAK'ed request. The MSN won't include the NAK'ed 1151f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * request but will include an ACK'ed request(s). 1152f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1153f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ack_psn = psn; 1154f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (aeth >> 29) 1155f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ack_psn--; 1156f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_acked); 1157f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp = to_iport(qp->ibqp.device, qp->port_num); 1158f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1159f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1160f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * The MSN might be for a later WQE than the PSN indicates so 1161f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * only complete WQEs that the PSN finishes. 1162f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1163f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell while ((diff = qib_cmp24(ack_psn, wqe->lpsn)) >= 0) { 1164f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1165f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * RDMA_READ_RESPONSE_ONLY is a special case since 1166f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * we want to generate completion events for everything 1167f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * before the RDMA read, copy the data, then generate 1168f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * the completion for the read. 1169f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1170f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_RDMA_READ && 1171f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(RDMA_READ_RESPONSE_ONLY) && 1172f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell diff == 0) { 1173f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ret = 1; 1174f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 1175f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1176f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1177f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If this request is a RDMA read or atomic, and the ACK is 1178f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * for a later operation, this ACK NAKs the RDMA read or 1179f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * atomic. In other words, only a RDMA_READ_LAST or ONLY 1180f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * can ACK a RDMA read and likewise for atomic ops. Note 1181f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * that the NAK case can only happen if relaxed ordering is 1182f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * used and requests are sent after an RDMA read or atomic 1183f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * is sent but before the response is received. 1184f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1185f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if ((wqe->wr.opcode == IB_WR_RDMA_READ && 1186f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (opcode != OP(RDMA_READ_RESPONSE_LAST) || diff != 0)) || 1187f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 1188f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) && 1189f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (opcode != OP(ATOMIC_ACKNOWLEDGE) || diff != 0))) { 1190f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Retry this request. */ 1191f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(qp->r_flags & QIB_R_RDMAR_SEQ)) { 1192f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_flags |= QIB_R_RDMAR_SEQ; 1193f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_restart_rc(qp, qp->s_last_psn + 1, 0); 1194f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (list_empty(&qp->rspwait)) { 1195f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_flags |= QIB_R_RSP_SEND; 1196f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_inc(&qp->refcount); 1197f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell list_add_tail(&qp->rspwait, 1198f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell &rcd->qp_wait_list); 1199f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1200f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1201f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1202f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * No need to process the ACK/NAK since we are 1203f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * restarting an earlier request. 1204f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1205f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 1206f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1207f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 1208f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 1209f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u64 *vaddr = wqe->sg_list[0].vaddr; 1210f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell *vaddr = val; 1211f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1212f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_num_rd_atomic && 1213f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (wqe->wr.opcode == IB_WR_RDMA_READ || 1214f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 1215f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) { 1216f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_num_rd_atomic--; 1217f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Restart sending task if fence is complete */ 1218f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if ((qp->s_flags & QIB_S_WAIT_FENCE) && 1219f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell !qp->s_num_rd_atomic) { 1220f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~(QIB_S_WAIT_FENCE | 1221f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell QIB_S_WAIT_ACK); 1222f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 1223f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else if (qp->s_flags & QIB_S_WAIT_RDMAR) { 1224f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~(QIB_S_WAIT_RDMAR | 1225f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell QIB_S_WAIT_ACK); 1226f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 1227f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1228f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1229f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = do_rc_completion(qp, wqe, ibp); 1230f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_acked == qp->s_tail) 1231f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1232f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1233f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1234f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (aeth >> 29) { 1235f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case 0: /* ACK */ 1236f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rc_acks++; 1237f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_acked != qp->s_tail) { 1238f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1239f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We are expecting more ACKs so 1240f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * reset the retransmit timer. 1241f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1242f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell start_timer(qp); 1243f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1244f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We can stop resending the earlier packets and 1245f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * continue with the next packet the receiver wants. 1246f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1247f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(qp->s_psn, psn) <= 0) 1248f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell reset_psn(qp, psn + 1); 1249f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else if (qib_cmp24(qp->s_psn, psn) <= 0) { 1250f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_LAST); 1251f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_psn = psn + 1; 1252f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1253f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & QIB_S_WAIT_ACK) { 1254f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~QIB_S_WAIT_ACK; 1255f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 1256f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1257f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_get_credit(qp, aeth); 1258f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_rnr_retry = qp->s_rnr_retry_cnt; 1259f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_retry = qp->s_retry_cnt; 1260f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell update_last_psn(qp, psn); 1261f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ret = 1; 1262f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 1263f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1264f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case 1: /* RNR NAK */ 1265f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rnr_naks++; 1266f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_acked == qp->s_tail) 1267f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 1268f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & QIB_S_WAIT_RNR) 1269f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 1270f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_rnr_retry == 0) { 1271f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell status = IB_WC_RNR_RETRY_EXC_ERR; 1272f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto class_b; 1273f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1274f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_rnr_retry_cnt < 7) 1275f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_rnr_retry--; 1276f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1277f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* The last valid PSN is the previous PSN. */ 1278f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell update_last_psn(qp, psn - 1); 1279f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1280f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK; 1281f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1282f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell reset_psn(qp, psn); 1283f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1284f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~(QIB_S_WAIT_SSN_CREDIT | QIB_S_WAIT_ACK); 1285f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_WAIT_RNR; 1286f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_timer.function = qib_rc_rnr_retry; 1287f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_timer.expires = jiffies + usecs_to_jiffies( 1288f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ib_qib_rnr_table[(aeth >> QIB_AETH_CREDIT_SHIFT) & 1289f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell QIB_AETH_CREDIT_MASK]); 1290f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell add_timer(&qp->s_timer); 1291f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 1292f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1293f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case 3: /* NAK */ 1294f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_acked == qp->s_tail) 1295f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 1296f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* The last valid PSN is the previous PSN. */ 1297f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell update_last_psn(qp, psn - 1); 1298f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch ((aeth >> QIB_AETH_CREDIT_SHIFT) & 1299f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell QIB_AETH_CREDIT_MASK) { 1300f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case 0: /* PSN sequence error */ 1301f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_seq_naks++; 1302f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1303f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Back up to the responder's expected PSN. 1304f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Note that we might get a NAK in the middle of an 1305f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * RDMA READ response which terminates the RDMA 1306f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * READ. 1307f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1308f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_restart_rc(qp, psn, 0); 1309f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 1310f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1311f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1312f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case 1: /* Invalid Request */ 1313f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell status = IB_WC_REM_INV_REQ_ERR; 1314f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_other_naks++; 1315f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto class_b; 1316f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1317f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case 2: /* Remote Access Error */ 1318f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell status = IB_WC_REM_ACCESS_ERR; 1319f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_other_naks++; 1320f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto class_b; 1321f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1322f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case 3: /* Remote Operation Error */ 1323f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell status = IB_WC_REM_OP_ERR; 1324f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_other_naks++; 1325f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellclass_b: 1326f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_last == qp->s_acked) { 1327f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_send_complete(qp, wqe, status); 1328f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_error_qp(qp, IB_WC_WR_FLUSH_ERR); 1329f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1330f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1331f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1332f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: 1333f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Ignore other reserved NAK error codes */ 1334f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto reserved; 1335f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1336f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_retry = qp->s_retry_cnt; 1337f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_rnr_retry = qp->s_rnr_retry_cnt; 1338f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 1339f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1340f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: /* 2: reserved */ 1341f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellreserved: 1342f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Ignore reserved NAK codes. */ 1343f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 1344f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1345f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1346f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellbail: 1347f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return ret; 1348f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 1349f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1350f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/* 1351f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We have seen an out of sequence RDMA read middle or last packet. 1352f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This ACKs SENDs and RDMA writes up to the first RDMA read or atomic SWQE. 1353f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1354f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic void rdma_seq_err(struct qib_qp *qp, struct qib_ibport *ibp, u32 psn, 1355f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ctxtdata *rcd) 1356f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 1357f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_swqe *wqe; 1358f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1359f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Remove QP from retry timer */ 1360f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & (QIB_S_TIMER | QIB_S_WAIT_RNR)) { 1361f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~(QIB_S_TIMER | QIB_S_WAIT_RNR); 1362f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell del_timer(&qp->s_timer); 1363f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1364f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1365f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_acked); 1366f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1367f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell while (qib_cmp24(psn, wqe->lpsn) > 0) { 1368f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_RDMA_READ || 1369f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 1370f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) 1371f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1372f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = do_rc_completion(qp, wqe, ibp); 1373f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1374f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1375f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rdma_seq++; 1376f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_flags |= QIB_R_RDMAR_SEQ; 1377f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_restart_rc(qp, qp->s_last_psn + 1, 0); 1378f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (list_empty(&qp->rspwait)) { 1379f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_flags |= QIB_R_RSP_SEND; 1380f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_inc(&qp->refcount); 1381f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 1382f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1383f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 1384f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1385f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/** 1386f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * qib_rc_rcv_resp - process an incoming RC response packet 1387f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @ibp: the port this packet came in on 1388f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @ohdr: the other headers for this packet 1389f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @data: the packet data 1390f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @tlen: the packet length 1391f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @qp: the QP for this packet 1392f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @opcode: the opcode for this packet 1393f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @psn: the packet sequence number for this packet 1394f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @hdrsize: the header length 1395f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @pmtu: the path MTU 1396f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 1397f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This is called from qib_rc_rcv() to process an incoming RC response 1398f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * packet for the given QP. 1399f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Called at interrupt level. 1400f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1401f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic void qib_rc_rcv_resp(struct qib_ibport *ibp, 1402f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_other_headers *ohdr, 1403f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell void *data, u32 tlen, 1404f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_qp *qp, 1405f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 opcode, 1406f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 psn, u32 hdrsize, u32 pmtu, 1407f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ctxtdata *rcd) 1408f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 1409f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_swqe *wqe; 1410dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn struct qib_pportdata *ppd = ppd_from_ibp(ibp); 1411f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell enum ib_wc_status status; 1412f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned long flags; 1413f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int diff; 1414f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 pad; 1415f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 aeth; 1416f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u64 val; 1417f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1418dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn if (opcode != OP(RDMA_READ_RESPONSE_MIDDLE)) { 1419dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn /* 1420dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn * If ACK'd PSN on SDMA busy list try to make progress to 1421dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn * reclaim SDMA credits. 1422dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn */ 1423dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn if ((qib_cmp24(psn, qp->s_sending_psn) >= 0) && 1424dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) { 1425dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn 1426dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn /* 1427dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn * If send tasklet not running attempt to progress 1428dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn * SDMA queue. 1429dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn */ 1430dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn if (!(qp->s_flags & QIB_S_BUSY)) { 1431dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn /* Acquire SDMA Lock */ 1432dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn spin_lock_irqsave(&ppd->sdma_lock, flags); 1433dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn /* Invoke sdma make progress */ 1434dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn qib_sdma_make_progress(ppd); 1435dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn /* Release SDMA Lock */ 1436dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn spin_unlock_irqrestore(&ppd->sdma_lock, flags); 1437dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn } 1438dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn } 1439dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn } 1440dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn 1441f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 1442414ed90cee32486c50f91b28990443e0dc21c868Mike Marciniszyn if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) 1443414ed90cee32486c50f91b28990443e0dc21c868Mike Marciniszyn goto ack_done; 1444f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1445f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Ignore invalid responses. */ 1446f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(psn, qp->s_next_psn) >= 0) 1447f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_done; 1448f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1449f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Ignore duplicate responses. */ 1450f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell diff = qib_cmp24(psn, qp->s_last_psn); 1451f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(diff <= 0)) { 1452f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Update credits for "ghost" ACKs */ 1453f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (diff == 0 && opcode == OP(ACKNOWLEDGE)) { 1454f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell aeth = be32_to_cpu(ohdr->u.aeth); 1455f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if ((aeth >> 29) == 0) 1456f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_get_credit(qp, aeth); 1457f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1458f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_done; 1459f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1460f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1461f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1462f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Skip everything other than the PSN we expect, if we are waiting 1463f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * for a reply to a restarted RDMA read or atomic op. 1464f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1465f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->r_flags & QIB_R_RDMAR_SEQ) { 1466f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(psn, qp->s_last_psn + 1) != 0) 1467f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_done; 1468f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_flags &= ~QIB_R_RDMAR_SEQ; 1469f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1470f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1471f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(qp->s_acked == qp->s_tail)) 1472f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_done; 1473f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_acked); 1474f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell status = IB_WC_SUCCESS; 1475f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1476f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (opcode) { 1477f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(ACKNOWLEDGE): 1478f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(ATOMIC_ACKNOWLEDGE): 1479f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_FIRST): 1480f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell aeth = be32_to_cpu(ohdr->u.aeth); 1481f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode == OP(ATOMIC_ACKNOWLEDGE)) { 1482f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell __be32 *p = ohdr->u.at.atomic_ack_eth; 1483f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1484f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell val = ((u64) be32_to_cpu(p[0]) << 32) | 1485f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell be32_to_cpu(p[1]); 1486f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else 1487f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell val = 0; 1488f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) || 1489f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode != OP(RDMA_READ_RESPONSE_FIRST)) 1490f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_done; 1491f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdrsize += 4; 1492f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_acked); 1493f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 1494f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_op_err; 1495f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1496f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If this is a response to a resent RDMA read, we 1497f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * have to be careful to copy the data to the right 1498f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * location. 1499f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1500f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, 1501f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe, psn, pmtu); 1502f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto read_middle; 1503f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1504f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_MIDDLE): 1505f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* no AETH, no ACK */ 1506f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1))) 1507f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_seq_err; 1508f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 1509f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_op_err; 1510f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellread_middle: 1511f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(tlen != (hdrsize + pmtu + 4))) 1512f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_len_err; 1513f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(pmtu >= qp->s_rdma_read_len)) 1514f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_len_err; 1515f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1516f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1517f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We got a response so update the timeout. 1518f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 4.096 usec. * (1 << qp->timeout) 1519f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1520f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_TIMER; 1521f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell mod_timer(&qp->s_timer, jiffies + 1522f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell usecs_to_jiffies((4096UL * (1UL << qp->timeout)) / 1523f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1000UL)); 1524f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & QIB_S_WAIT_ACK) { 1525f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~QIB_S_WAIT_ACK; 1526f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 1527f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1528f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1529f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode == OP(RDMA_READ_RESPONSE_MIDDLE)) 1530f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_retry = qp->s_retry_cnt; 1531f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1532f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1533f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Update the RDMA receive state but do the copy w/o 1534f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * holding the locks and blocking interrupts. 1535f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1536f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_rdma_read_len -= pmtu; 1537f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell update_last_psn(qp, psn); 1538f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 1539f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_copy_sge(&qp->s_rdma_read_sge, data, pmtu, 0); 1540f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 1541f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1542f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_ONLY): 1543f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell aeth = be32_to_cpu(ohdr->u.aeth); 1544f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd)) 1545f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_done; 1546f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Get the number of bytes the message was padded by. */ 1547f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 1548f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1549f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Check that the data size is >= 0 && <= pmtu. 1550f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Remember to account for the AETH header (4) and 1551f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * ICRC (4). 1552f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1553f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(tlen < (hdrsize + pad + 8))) 1554f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_len_err; 1555f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1556f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If this is a response to a resent RDMA read, we 1557f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * have to be careful to copy the data to the right 1558f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * location. 1559f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1560f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_acked); 1561f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, 1562f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe, psn, pmtu); 1563f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto read_last; 1564f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1565f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_LAST): 1566f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* ACKs READ req. */ 1567f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1))) 1568f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_seq_err; 1569f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 1570f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_op_err; 1571f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Get the number of bytes the message was padded by. */ 1572f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 1573f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1574f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Check that the data size is >= 1 && <= pmtu. 1575f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Remember to account for the AETH header (4) and 1576f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * ICRC (4). 1577f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1578f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(tlen <= (hdrsize + pad + 8))) 1579f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_len_err; 1580f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellread_last: 1581f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell tlen -= hdrsize + pad + 8; 1582f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(tlen != qp->s_rdma_read_len)) 1583f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_len_err; 1584f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell aeth = be32_to_cpu(ohdr->u.aeth); 1585f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_copy_sge(&qp->s_rdma_read_sge, data, tlen, 0); 1586f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell WARN_ON(qp->s_rdma_read_sge.num_sge); 1587f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (void) do_rc_ack(qp, aeth, psn, 1588f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell OP(RDMA_READ_RESPONSE_LAST), 0, rcd); 1589f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_done; 1590f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1591f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1592f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellack_op_err: 1593f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell status = IB_WC_LOC_QP_OP_ERR; 1594f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_err; 1595f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1596f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellack_seq_err: 1597f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell rdma_seq_err(qp, ibp, psn, rcd); 1598f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_done; 1599f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1600f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellack_len_err: 1601f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell status = IB_WC_LOC_LEN_ERR; 1602f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellack_err: 1603f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_last == qp->s_acked) { 1604f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_send_complete(qp, wqe, status); 1605f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_error_qp(qp, IB_WC_WR_FLUSH_ERR); 1606f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1607f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellack_done: 1608f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 1609f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellbail: 1610f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return; 1611f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 1612f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1613f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/** 1614f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * qib_rc_rcv_error - process an incoming duplicate or error RC packet 1615f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @ohdr: the other headers for this packet 1616f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @data: the packet data 1617f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @qp: the QP for this packet 1618f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @opcode: the opcode for this packet 1619f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @psn: the packet sequence number for this packet 1620f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @diff: the difference between the PSN and the expected PSN 1621f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 1622f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This is called from qib_rc_rcv() to process an unexpected 1623f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * incoming RC packet for the given QP. 1624f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Called at interrupt level. 1625f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Return 1 if no more processing is needed; otherwise return 0 to 1626f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * schedule a response to be sent. 1627f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1628f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic int qib_rc_rcv_error(struct qib_other_headers *ohdr, 1629f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell void *data, 1630f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_qp *qp, 1631f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 opcode, 1632f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 psn, 1633f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int diff, 1634f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ctxtdata *rcd) 1635f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 1636f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); 1637f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ack_entry *e; 1638f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned long flags; 1639f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u8 i, prev; 1640f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int old_req; 1641f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1642f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (diff > 0) { 1643f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1644f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Packet sequence error. 1645f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * A NAK will ACK earlier sends and RDMA writes. 1646f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Don't queue the NAK if we already sent one. 1647f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1648f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!qp->r_nak_state) { 1649f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rc_seqnak++; 1650f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = IB_NAK_PSN_ERROR; 1651f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Use the expected PSN. */ 1652f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_ack_psn = qp->r_psn; 1653f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1654f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Wait to send the sequence NAK until all packets 1655f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * in the receive queue have been processed. 1656f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Otherwise, we end up propagating congestion. 1657f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1658f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (list_empty(&qp->rspwait)) { 1659f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_flags |= QIB_R_RSP_NAK; 1660f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_inc(&qp->refcount); 1661f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 1662f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1663f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1664f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto done; 1665f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1666f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1667f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1668f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Handle a duplicate request. Don't re-execute SEND, RDMA 1669f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * write or atomic op. Don't NAK errors, just silently drop 1670f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * the duplicate request. Note that r_sge, r_len, and 1671f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * r_rcv_len may be in use so don't modify them. 1672f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 1673f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We are supposed to ACK the earliest duplicate PSN but we 1674f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * can coalesce an outstanding duplicate ACK. We have to 1675f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * send the earliest so that RDMA reads can be restarted at 1676f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * the requester's expected PSN. 1677f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 1678f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * First, find where this duplicate PSN falls within the 1679f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * ACKs previously sent. 1680f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * old_req is true if there is an older response that is scheduled 1681f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * to be sent before sending this one. 1682f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1683f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e = NULL; 1684f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell old_req = 1; 1685f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rc_dupreq++; 1686f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1687f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 1688f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1689f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell for (i = qp->r_head_ack_queue; ; i = prev) { 1690f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (i == qp->s_tail_ack_queue) 1691f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell old_req = 0; 1692f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (i) 1693f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell prev = i - 1; 1694f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else 1695f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell prev = QIB_MAX_RDMA_ATOMIC; 1696f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (prev == qp->r_head_ack_queue) { 1697f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e = NULL; 1698f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1699f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1700f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e = &qp->s_ack_queue[prev]; 1701f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!e->opcode) { 1702f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e = NULL; 1703f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1704f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1705f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(psn, e->psn) >= 0) { 1706f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (prev == qp->s_tail_ack_queue && 1707f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_cmp24(psn, e->lpsn) <= 0) 1708f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell old_req = 0; 1709f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1710f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1711f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1712f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (opcode) { 1713f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_REQUEST): { 1714f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct ib_reth *reth; 1715f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 offset; 1716f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 len; 1717f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1718f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1719f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If we didn't find the RDMA read request in the ack queue, 1720f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * we can ignore this request. 1721f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1722f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!e || e->opcode != OP(RDMA_READ_REQUEST)) 1723f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto unlock_done; 1724f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* RETH comes after BTH */ 1725f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell reth = &ohdr->u.rc.reth; 1726f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1727f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Address range must be a subset of the original 1728f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * request and start on pmtu boundaries. 1729f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We reuse the old ack_queue slot since the requester 1730f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * should not back up and request an earlier PSN for the 1731f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * same request. 1732f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1733f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell offset = ((psn - e->psn) & QIB_PSN_MASK) * 1734f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ib_mtu_enum_to_int(qp->path_mtu); 1735f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = be32_to_cpu(reth->length); 1736f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(offset + len != e->rdma_sge.sge_length)) 1737f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto unlock_done; 1738f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (e->rdma_sge.mr) { 1739f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_dec(&e->rdma_sge.mr->refcount); 1740f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.mr = NULL; 1741f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1742f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len != 0) { 1743f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 rkey = be32_to_cpu(reth->rkey); 1744f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u64 vaddr = be64_to_cpu(reth->vaddr); 1745f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int ok; 1746f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1747f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ok = qib_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey, 1748f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell IB_ACCESS_REMOTE_READ); 1749f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(!ok)) 1750f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto unlock_done; 1751f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else { 1752f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.vaddr = NULL; 1753f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.length = 0; 1754f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.sge_length = 0; 1755f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1756f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->psn = psn; 1757f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (old_req) 1758f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto unlock_done; 1759f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_tail_ack_queue = prev; 1760f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1761f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1762f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1763f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(COMPARE_SWAP): 1764f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(FETCH_ADD): { 1765f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1766f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If we didn't find the atomic request in the ack queue 1767f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * or the send tasklet is already backed up to send an 1768f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * earlier entry, we can ignore this request. 1769f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1770f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!e || e->opcode != (u8) opcode || old_req) 1771f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto unlock_done; 1772f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_tail_ack_queue = prev; 1773f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1774f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1775f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1776f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: 1777f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1778f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Ignore this operation if it doesn't request an ACK 1779f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * or an earlier RDMA read or atomic is going to be resent. 1780f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1781f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(psn & IB_BTH_REQ_ACK) || old_req) 1782f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto unlock_done; 1783f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1784f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Resend the most recent ACK if this request is 1785f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * after all the previous RDMA reads and atomics. 1786f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1787f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (i == qp->r_head_ack_queue) { 1788f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 1789f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = 0; 1790f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_ack_psn = qp->r_psn - 1; 1791f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto send_ack; 1792f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1793f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1794f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Try to send a simple ACK to work around a Mellanox bug 1795f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * which doesn't accept a RDMA read response or atomic 1796f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * response as an ACK for earlier SENDs or RDMA writes. 1797f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1798f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(qp->s_flags & QIB_S_RESP_PENDING)) { 1799f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 1800f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = 0; 1801f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_ack_psn = qp->s_ack_queue[i].psn - 1; 1802f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto send_ack; 1803f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1804f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1805f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Resend the RDMA read or atomic op which 1806f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * ACKs this duplicate request. 1807f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1808f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_tail_ack_queue = i; 1809f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1810f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1811f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_state = OP(ACKNOWLEDGE); 1812f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_RESP_PENDING; 1813f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = 0; 1814f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 1815f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1816f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellunlock_done: 1817f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 1818f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbelldone: 1819f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return 1; 1820f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1821f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellsend_ack: 1822f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return 0; 1823f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 1824f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1825f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellvoid qib_rc_error(struct qib_qp *qp, enum ib_wc_status err) 1826f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 1827f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned long flags; 1828f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int lastwqe; 1829f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1830f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 1831f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell lastwqe = qib_error_qp(qp, err); 1832f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 1833f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1834f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (lastwqe) { 1835f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct ib_event ev; 1836f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1837f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ev.device = qp->ibqp.device; 1838f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ev.element.qp = &qp->ibqp; 1839f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ev.event = IB_EVENT_QP_LAST_WQE_REACHED; 1840f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); 1841f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1842f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 1843f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1844f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic inline void qib_update_ack_queue(struct qib_qp *qp, unsigned n) 1845f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 1846f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned next; 1847f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1848f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell next = n + 1; 1849f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (next > QIB_MAX_RDMA_ATOMIC) 1850f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell next = 0; 1851f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_tail_ack_queue = next; 1852f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_state = OP(ACKNOWLEDGE); 1853f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 1854f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1855f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/** 1856f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * qib_rc_rcv - process an incoming RC packet 1857f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @rcd: the context pointer 1858f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @hdr: the header of this packet 1859f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @has_grh: true if the header has a GRH 1860f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @data: the packet data 1861f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @tlen: the packet length 1862f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @qp: the QP for this packet 1863f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 1864f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This is called from qib_qp_rcv() to process an incoming RC packet 1865f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * for the given QP. 1866f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Called at interrupt level. 1867f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1868f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellvoid qib_rc_rcv(struct qib_ctxtdata *rcd, struct qib_ib_header *hdr, 1869f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int has_grh, void *data, u32 tlen, struct qib_qp *qp) 1870f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 1871f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ibport *ibp = &rcd->ppd->ibport_data; 1872f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_other_headers *ohdr; 1873f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 opcode; 1874f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 hdrsize; 1875f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 psn; 1876f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 pad; 1877f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct ib_wc wc; 1878f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu); 1879f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int diff; 1880f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct ib_reth *reth; 1881f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned long flags; 1882f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int ret; 1883f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1884f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check for GRH */ 1885f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!has_grh) { 1886f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr = &hdr->u.oth; 1887f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdrsize = 8 + 12; /* LRH + BTH */ 1888f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else { 1889f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr = &hdr->u.l.oth; 1890f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdrsize = 8 + 40 + 12; /* LRH + GRH + BTH */ 1891f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1892f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1893f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode = be32_to_cpu(ohdr->bth[0]); 1894f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 1895f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_ruc_check_hdr(ibp, hdr, has_grh, qp, opcode)) 1896f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto sunlock; 1897f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 1898f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1899f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell psn = be32_to_cpu(ohdr->bth[2]); 1900f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode >>= 24; 1901f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1902f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1903f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Process responses (ACKs) before anything else. Note that the 1904f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * packet sequence number will be for something in the send work 1905f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * queue rather than the expected receive packet sequence number. 1906f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * In other words, this QP is the requester. 1907f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1908f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) && 1909f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode <= OP(ATOMIC_ACKNOWLEDGE)) { 1910f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_rc_rcv_resp(ibp, ohdr, data, tlen, qp, opcode, psn, 1911f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdrsize, pmtu, rcd); 1912a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell return; 1913f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1914f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1915f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Compute 24 bits worth of difference. */ 1916f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell diff = qib_cmp24(psn, qp->r_psn); 1917f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(diff)) { 1918f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_rc_rcv_error(ohdr, data, qp, opcode, psn, diff, rcd)) 1919a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell return; 1920f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto send_ack; 1921f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1922f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1923f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check for opcode sequence errors. */ 1924f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (qp->r_state) { 1925f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_FIRST): 1926f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_MIDDLE): 1927f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode == OP(SEND_MIDDLE) || 1928f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(SEND_LAST) || 1929f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(SEND_LAST_WITH_IMMEDIATE)) 1930f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1931f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 1932f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1933f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_FIRST): 1934f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_MIDDLE): 1935f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode == OP(RDMA_WRITE_MIDDLE) || 1936f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(RDMA_WRITE_LAST) || 1937f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) 1938f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1939f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 1940f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1941f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: 1942f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode == OP(SEND_MIDDLE) || 1943f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(SEND_LAST) || 1944f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(SEND_LAST_WITH_IMMEDIATE) || 1945f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(RDMA_WRITE_MIDDLE) || 1946f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(RDMA_WRITE_LAST) || 1947f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) 1948f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 1949f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1950f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Note that it is up to the requester to not send a new 1951f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * RDMA read or atomic operation before receiving an ACK 1952f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * for the previous operation. 1953f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1954f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1955f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1956f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1957f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell memset(&wc, 0, sizeof wc); 1958f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1959f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->state == IB_QPS_RTR && !(qp->r_flags & QIB_R_COMM_EST)) { 1960f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_flags |= QIB_R_COMM_EST; 1961f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->ibqp.event_handler) { 1962f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct ib_event ev; 1963f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1964f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ev.device = qp->ibqp.device; 1965f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ev.element.qp = &qp->ibqp; 1966f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ev.event = IB_EVENT_COMM_EST; 1967f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); 1968f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1969f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1970f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1971f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* OK, process the packet. */ 1972f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (opcode) { 1973f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_FIRST): 1974f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ret = qib_get_rwqe(qp, 0); 1975f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (ret < 0) 1976f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_op_err; 1977f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!ret) 1978f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto rnr_nak; 1979f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_rcv_len = 0; 1980f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 1981f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_MIDDLE): 1982f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_MIDDLE): 1983f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellsend_middle: 1984f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check for invalid length PMTU or posted rwqe len. */ 1985f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(tlen != (hdrsize + pmtu + 4))) 1986f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 1987f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_rcv_len += pmtu; 1988f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(qp->r_rcv_len > qp->r_len)) 1989f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 1990f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_copy_sge(&qp->r_sge, data, pmtu, 1); 1991f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1992f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1993f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE): 1994f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* consume RWQE */ 1995f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ret = qib_get_rwqe(qp, 1); 1996f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (ret < 0) 1997f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_op_err; 1998f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!ret) 1999f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto rnr_nak; 2000f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto send_last_imm; 2001f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2002f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_ONLY): 2003f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_ONLY_WITH_IMMEDIATE): 2004f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ret = qib_get_rwqe(qp, 0); 2005f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (ret < 0) 2006f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_op_err; 2007f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!ret) 2008f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto rnr_nak; 2009f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_rcv_len = 0; 2010f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode == OP(SEND_ONLY)) 2011f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto send_last; 2012f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 2013f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_LAST_WITH_IMMEDIATE): 2014f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellsend_last_imm: 2015f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.ex.imm_data = ohdr->u.imm_data; 2016f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdrsize += 4; 2017f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.wc_flags = IB_WC_WITH_IMM; 2018f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 2019f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_LAST): 2020f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_LAST): 2021f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellsend_last: 2022f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Get the number of bytes the message was padded by. */ 2023f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 2024f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check for invalid length. */ 2025f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* XXX LAST len should be >= 1 */ 2026f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(tlen < (hdrsize + pad + 4))) 2027f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 2028f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Don't count the CRC. */ 2029f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell tlen -= (hdrsize + pad + 4); 2030f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.byte_len = tlen + qp->r_rcv_len; 2031f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(wc.byte_len > qp->r_len)) 2032f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 2033f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_copy_sge(&qp->r_sge, data, tlen, 1); 2034f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell while (qp->r_sge.num_sge) { 2035f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_dec(&qp->r_sge.sge.mr->refcount); 2036f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (--qp->r_sge.num_sge) 2037f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.sge = *qp->r_sge.sg_list++; 2038f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2039f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_msn++; 2040f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!test_and_clear_bit(QIB_R_WRID_VALID, &qp->r_aflags)) 2041f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 2042f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.wr_id = qp->r_wr_id; 2043f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.status = IB_WC_SUCCESS; 2044f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE) || 2045f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE)) 2046f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.opcode = IB_WC_RECV_RDMA_WITH_IMM; 2047f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else 2048f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.opcode = IB_WC_RECV; 2049f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.qp = &qp->ibqp; 2050f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.src_qp = qp->remote_qpn; 2051f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.slid = qp->remote_ah_attr.dlid; 2052f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.sl = qp->remote_ah_attr.sl; 2053f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Signal completion event if the solicited bit is set. */ 2054f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 2055f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (ohdr->bth[0] & 2056f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be32(IB_BTH_SOLICITED)) != 0); 2057f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 2058f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2059f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_FIRST): 2060f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_ONLY): 2061f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): 2062f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) 2063f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 2064f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* consume RWQE */ 2065f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell reth = &ohdr->u.rc.reth; 2066f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdrsize += sizeof(*reth); 2067f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_len = be32_to_cpu(reth->length); 2068f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_rcv_len = 0; 2069f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.sg_list = NULL; 2070f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->r_len != 0) { 2071f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 rkey = be32_to_cpu(reth->rkey); 2072f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u64 vaddr = be64_to_cpu(reth->vaddr); 2073f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int ok; 2074f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2075f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check rkey & NAK */ 2076f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ok = qib_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr, 2077f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell rkey, IB_ACCESS_REMOTE_WRITE); 2078f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(!ok)) 2079f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_acc; 2080f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.num_sge = 1; 2081f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else { 2082f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.num_sge = 0; 2083f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.sge.mr = NULL; 2084f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.sge.vaddr = NULL; 2085f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.sge.length = 0; 2086f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.sge.sge_length = 0; 2087f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2088f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode == OP(RDMA_WRITE_FIRST)) 2089f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto send_middle; 2090f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else if (opcode == OP(RDMA_WRITE_ONLY)) 2091f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto send_last; 2092f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ret = qib_get_rwqe(qp, 1); 2093f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (ret < 0) 2094f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_op_err; 2095f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!ret) 2096f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto rnr_nak; 20975715f5d44b93507693feca6c62290e123e5ef688Jason Gunthorpe wc.ex.imm_data = ohdr->u.rc.imm_data; 20985715f5d44b93507693feca6c62290e123e5ef688Jason Gunthorpe hdrsize += 4; 20995715f5d44b93507693feca6c62290e123e5ef688Jason Gunthorpe wc.wc_flags = IB_WC_WITH_IMM; 21005715f5d44b93507693feca6c62290e123e5ef688Jason Gunthorpe goto send_last; 2101f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2102f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_REQUEST): { 2103f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ack_entry *e; 2104f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 len; 2105f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u8 next; 2106f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2107f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) 2108f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 2109f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell next = qp->r_head_ack_queue + 1; 2110f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* s_ack_queue is size QIB_MAX_RDMA_ATOMIC+1 so use > not >= */ 2111f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (next > QIB_MAX_RDMA_ATOMIC) 2112f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell next = 0; 2113f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 2114f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(next == qp->s_tail_ack_queue)) { 2115f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!qp->s_ack_queue[next].sent) 2116f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv_unlck; 2117f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_update_ack_queue(qp, next); 2118f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2119f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e = &qp->s_ack_queue[qp->r_head_ack_queue]; 2120f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (e->opcode == OP(RDMA_READ_REQUEST) && e->rdma_sge.mr) { 2121f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_dec(&e->rdma_sge.mr->refcount); 2122f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.mr = NULL; 2123f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2124f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell reth = &ohdr->u.rc.reth; 2125f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = be32_to_cpu(reth->length); 2126f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len) { 2127f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 rkey = be32_to_cpu(reth->rkey); 2128f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u64 vaddr = be64_to_cpu(reth->vaddr); 2129f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int ok; 2130f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2131f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check rkey & NAK */ 2132f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ok = qib_rkey_ok(qp, &e->rdma_sge, len, vaddr, 2133f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell rkey, IB_ACCESS_REMOTE_READ); 2134f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(!ok)) 2135f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_acc_unlck; 2136f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 2137f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Update the next expected PSN. We add 1 later 2138f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * below, so only add the remainder here. 2139f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 2140f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len > pmtu) 2141f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_psn += (len - 1) / pmtu; 2142f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else { 2143f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.mr = NULL; 2144f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.vaddr = NULL; 2145f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.length = 0; 2146f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.sge_length = 0; 2147f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2148f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->opcode = opcode; 2149f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->sent = 0; 2150f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->psn = psn; 2151f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->lpsn = qp->r_psn; 2152f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 2153f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We need to increment the MSN here instead of when we 2154f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * finish sending the result since a duplicate request would 2155f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * increment it more than once. 2156f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 2157f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_msn++; 2158f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_psn++; 2159f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_state = opcode; 2160f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = 0; 2161f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_head_ack_queue = next; 2162f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2163f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Schedule the send tasklet. */ 2164f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_RESP_PENDING; 2165f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 2166f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2167a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell goto sunlock; 2168f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2169f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2170f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(COMPARE_SWAP): 2171f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(FETCH_ADD): { 2172f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct ib_atomic_eth *ateth; 2173f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ack_entry *e; 2174f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u64 vaddr; 2175f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic64_t *maddr; 2176f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u64 sdata; 2177f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 rkey; 2178f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u8 next; 2179f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2180f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC))) 2181f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 2182f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell next = qp->r_head_ack_queue + 1; 2183f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (next > QIB_MAX_RDMA_ATOMIC) 2184f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell next = 0; 2185f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 2186f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(next == qp->s_tail_ack_queue)) { 2187f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!qp->s_ack_queue[next].sent) 2188f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv_unlck; 2189f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_update_ack_queue(qp, next); 2190f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2191f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e = &qp->s_ack_queue[qp->r_head_ack_queue]; 2192f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (e->opcode == OP(RDMA_READ_REQUEST) && e->rdma_sge.mr) { 2193f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_dec(&e->rdma_sge.mr->refcount); 2194f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.mr = NULL; 2195f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2196f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ateth = &ohdr->u.atomic_eth; 2197f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell vaddr = ((u64) be32_to_cpu(ateth->vaddr[0]) << 32) | 2198f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell be32_to_cpu(ateth->vaddr[1]); 2199f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(vaddr & (sizeof(u64) - 1))) 2200f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv_unlck; 2201f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell rkey = be32_to_cpu(ateth->rkey); 2202f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check rkey & NAK */ 2203f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(!qib_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64), 2204f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell vaddr, rkey, 2205f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell IB_ACCESS_REMOTE_ATOMIC))) 2206f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_acc_unlck; 2207f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Perform atomic OP and save result. */ 2208f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell maddr = (atomic64_t *) qp->r_sge.sge.vaddr; 2209f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell sdata = be64_to_cpu(ateth->swap_data); 2210f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->atomic_data = (opcode == OP(FETCH_ADD)) ? 2211f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (u64) atomic64_add_return(sdata, maddr) - sdata : 2212f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr, 2213f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell be64_to_cpu(ateth->compare_data), 2214f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell sdata); 2215f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_dec(&qp->r_sge.sge.mr->refcount); 2216f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.num_sge = 0; 2217f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->opcode = opcode; 2218f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->sent = 0; 2219f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->psn = psn; 2220f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->lpsn = psn; 2221f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_msn++; 2222f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_psn++; 2223f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_state = opcode; 2224f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = 0; 2225f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_head_ack_queue = next; 2226f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2227f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Schedule the send tasklet. */ 2228f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_RESP_PENDING; 2229f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 2230f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2231a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell goto sunlock; 2232f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2233f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2234f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: 2235f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* NAK unknown opcodes. */ 2236f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 2237f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2238f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_psn++; 2239f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_state = opcode; 2240f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_ack_psn = psn; 2241f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = 0; 2242f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Send an ACK if requested or required. */ 2243f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (psn & (1 << 31)) 2244f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto send_ack; 2245a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell return; 2246f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2247f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellrnr_nak: 2248f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = IB_RNR_NAK | qp->r_min_rnr_timer; 2249f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_ack_psn = qp->r_psn; 2250f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Queue RNR NAK for later */ 2251f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (list_empty(&qp->rspwait)) { 2252f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_flags |= QIB_R_RSP_NAK; 2253f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_inc(&qp->refcount); 2254f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 2255f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2256a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell return; 2257f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2258f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellnack_op_err: 2259f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_rc_error(qp, IB_WC_LOC_QP_OP_ERR); 2260f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = IB_NAK_REMOTE_OPERATIONAL_ERROR; 2261f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_ack_psn = qp->r_psn; 2262f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Queue NAK for later */ 2263f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (list_empty(&qp->rspwait)) { 2264f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_flags |= QIB_R_RSP_NAK; 2265f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_inc(&qp->refcount); 2266f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 2267f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2268a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell return; 2269f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2270f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellnack_inv_unlck: 2271f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 2272f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellnack_inv: 2273f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_rc_error(qp, IB_WC_LOC_QP_OP_ERR); 2274f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = IB_NAK_INVALID_REQUEST; 2275f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_ack_psn = qp->r_psn; 2276f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Queue NAK for later */ 2277f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (list_empty(&qp->rspwait)) { 2278f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_flags |= QIB_R_RSP_NAK; 2279f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_inc(&qp->refcount); 2280f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 2281f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2282a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell return; 2283f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2284f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellnack_acc_unlck: 2285f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 2286f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellnack_acc: 2287f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_rc_error(qp, IB_WC_LOC_PROT_ERR); 2288f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; 2289f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_ack_psn = qp->r_psn; 2290f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellsend_ack: 2291f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_send_rc_ack(qp); 2292f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return; 2293f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2294f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellsunlock: 2295f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 2296f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 2297