1f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/* 2f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Copyright (c) 2006, 2007, 2008, 2009 QLogic Corporation. All rights reserved. 3f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. 4f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 5f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This software is available to you under a choice of one of two 6f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * licenses. You may choose to be licensed under the terms of the GNU 7f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * General Public License (GPL) Version 2, available from the file 8f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * COPYING in the main directory of this source tree, or the 9f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * OpenIB.org BSD license below: 10f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 11f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Redistribution and use in source and binary forms, with or 12f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * without modification, are permitted provided that the following 13f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * conditions are met: 14f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 15f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * - Redistributions of source code must retain the above 16f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * copyright notice, this list of conditions and the following 17f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * disclaimer. 18f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 19f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * - Redistributions in binary form must reproduce the above 20f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * copyright notice, this list of conditions and the following 21f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * disclaimer in the documentation and/or other materials 22f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * provided with the distribution. 23f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 24f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * SOFTWARE. 32f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 33f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 34f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell#include <linux/io.h> 35f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 36f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell#include "qib.h" 37f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 38f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/* cut down ridiculously long IB macro names */ 39f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell#define OP(x) IB_OPCODE_RC_##x 40f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 41f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic void rc_timeout(unsigned long arg); 42f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 43f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic u32 restart_sge(struct qib_sge_state *ss, struct qib_swqe *wqe, 44f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 psn, u32 pmtu) 45f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 46f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 len; 47f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 48f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = ((psn - wqe->psn) & QIB_PSN_MASK) * pmtu; 49f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ss->sge = wqe->sg_list[0]; 50f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ss->sg_list = wqe->sg_list + 1; 51f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ss->num_sge = wqe->wr.num_sge; 52f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ss->total_len = wqe->length; 53f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_skip_sge(ss, len, 0); 54f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return wqe->length - len; 55f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 56f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 57f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic void start_timer(struct qib_qp *qp) 58f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 59f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_TIMER; 60f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_timer.function = rc_timeout; 61f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 4.096 usec. * (1 << qp->timeout) */ 62d0f2faf72d51dacf5c5e8dec7dca22d0395896e2Mike Marciniszyn qp->s_timer.expires = jiffies + qp->timeout_jiffies; 63f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell add_timer(&qp->s_timer); 64f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 65f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 66f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/** 67f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * qib_make_rc_ack - construct a response packet (ACK, NAK, or RDMA read) 68f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @dev: the device for this QP 69f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @qp: a pointer to the QP 70f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @ohdr: a pointer to the IB header being constructed 71f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @pmtu: the path MTU 72f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 73f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Return 1 if constructed; otherwise, return 0. 74f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Note that we are in the responder's side of the QP context. 75f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Note the QP s_lock must be held. 76f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 77f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic int qib_make_rc_ack(struct qib_ibdev *dev, struct qib_qp *qp, 78f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_other_headers *ohdr, u32 pmtu) 79f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 80f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ack_entry *e; 81f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 hwords; 82f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 len; 83f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 bth0; 84f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 bth2; 85f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 86f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Don't send an ACK if we aren't supposed to. */ 87f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) 88f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 89f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 90f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* header size in 32-bit words LRH+BTH = (8+12)/4. */ 91f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords = 5; 92f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 93f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (qp->s_ack_state) { 94f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_LAST): 95f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_ONLY): 96f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e = &qp->s_ack_queue[qp->s_tail_ack_queue]; 97f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (e->rdma_sge.mr) { 986a82649f217023863d6b1740017e6c3dd6685327Mike Marciniszyn qib_put_mr(e->rdma_sge.mr); 99f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.mr = NULL; 100f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 101f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 102f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(ATOMIC_ACKNOWLEDGE): 103f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 104f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We can increment the tail pointer now that the last 105f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * response has been sent instead of only being 106f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * constructed. 107f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 108f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_tail_ack_queue > QIB_MAX_RDMA_ATOMIC) 109f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_tail_ack_queue = 0; 110f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 111f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_ONLY): 112f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(ACKNOWLEDGE): 113f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check for no next entry in the queue. */ 114f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->r_head_ack_queue == qp->s_tail_ack_queue) { 115f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & QIB_S_ACK_PENDING) 116f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto normal; 117f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 118f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 119f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 120f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e = &qp->s_ack_queue[qp->s_tail_ack_queue]; 121f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (e->opcode == OP(RDMA_READ_REQUEST)) { 122f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 123f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If a RDMA read response is being resent and 124f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * we haven't seen the duplicate request yet, 125f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * then stop sending the remaining responses the 126f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * responder has seen until the requester resends it. 127f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 128f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = e->rdma_sge.sge_length; 129f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len && !e->rdma_sge.mr) { 130f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_tail_ack_queue = qp->r_head_ack_queue; 131f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 132f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 133f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Copy SGE state in case we need to resend */ 134f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_rdma_mr = e->rdma_sge.mr; 135f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_rdma_mr) 1366a82649f217023863d6b1740017e6c3dd6685327Mike Marciniszyn qib_get_mr(qp->s_rdma_mr); 137f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_rdma_sge.sge = e->rdma_sge; 138f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_rdma_sge.num_sge = 1; 139f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur_sge = &qp->s_ack_rdma_sge; 140f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len > pmtu) { 141f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = pmtu; 142f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST); 143f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else { 144f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY); 145f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->sent = 1; 146f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 147f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.aeth = qib_compute_aeth(qp); 148f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords++; 149f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_rdma_psn = e->psn; 150f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK; 151f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else { 152f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* COMPARE_SWAP or FETCH_ADD */ 153f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur_sge = NULL; 154f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = 0; 155f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); 156f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.at.aeth = qib_compute_aeth(qp); 157f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.at.atomic_ack_eth[0] = 158f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be32(e->atomic_data >> 32); 159f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.at.atomic_ack_eth[1] = 160f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be32(e->atomic_data); 161f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += sizeof(ohdr->u.at) / sizeof(u32); 162f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 = e->psn & QIB_PSN_MASK; 163f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->sent = 1; 164f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 165f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 = qp->s_ack_state << 24; 166f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 167f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 168f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_FIRST): 169f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE); 170f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 171f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_MIDDLE): 172f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur_sge = &qp->s_ack_rdma_sge; 173f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_rdma_mr = qp->s_ack_rdma_sge.sge.mr; 174f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_rdma_mr) 1756a82649f217023863d6b1740017e6c3dd6685327Mike Marciniszyn qib_get_mr(qp->s_rdma_mr); 176f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = qp->s_ack_rdma_sge.sge.sge_length; 177f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len > pmtu) 178f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = pmtu; 179f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else { 180f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.aeth = qib_compute_aeth(qp); 181f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords++; 182f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST); 183f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e = &qp->s_ack_queue[qp->s_tail_ack_queue]; 184f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->sent = 1; 185f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 186f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 = qp->s_ack_state << 24; 187f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK; 188f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 189f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 190f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: 191f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellnormal: 192f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 193f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Send a regular ACK. 194f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Set the s_ack_state so we wait until after sending 195f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * the ACK before setting s_ack_state to ACKNOWLEDGE 196f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * (see above). 197f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 198f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_state = OP(SEND_ONLY); 199f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~QIB_S_ACK_PENDING; 200f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur_sge = NULL; 201f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_nak_state) 202f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.aeth = 203f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be32((qp->r_msn & QIB_MSN_MASK) | 204f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (qp->s_nak_state << 205f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell QIB_AETH_CREDIT_SHIFT)); 206f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else 207f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.aeth = qib_compute_aeth(qp); 208f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords++; 209f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = 0; 210f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 = OP(ACKNOWLEDGE) << 24; 211f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 = qp->s_ack_psn & QIB_PSN_MASK; 212f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 213f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_rdma_ack_cnt++; 214f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_hdrwords = hwords; 215f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur_size = len; 216f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_make_ruc_header(qp, ohdr, bth0, bth2); 217f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return 1; 218f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 219f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellbail: 220f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_state = OP(ACKNOWLEDGE); 221f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~(QIB_S_RESP_PENDING | QIB_S_ACK_PENDING); 222f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return 0; 223f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 224f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 225f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/** 226f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * qib_make_rc_req - construct a request packet (SEND, RDMA r/w, ATOMIC) 227f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @qp: a pointer to the QP 228f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 229f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Return 1 if constructed; otherwise, return 0. 230f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 231f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellint qib_make_rc_req(struct qib_qp *qp) 232f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 233f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ibdev *dev = to_idev(qp->ibqp.device); 234f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_other_headers *ohdr; 235f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_sge_state *ss; 236f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_swqe *wqe; 237f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 hwords; 238f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 len; 239f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 bth0; 240f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 bth2; 241cc6ea1385b43487f6ef03bdc91416c8366d28311Mike Marciniszyn u32 pmtu = qp->pmtu; 242f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell char newreq; 243f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned long flags; 244f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int ret = 0; 245f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int delta; 246f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2471c94283ddbe8a9945c4aaac8b0be90d47f97f2dfMike Marciniszyn ohdr = &qp->s_hdr->u.oth; 248f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->remote_ah_attr.ah_flags & IB_AH_GRH) 2491c94283ddbe8a9945c4aaac8b0be90d47f97f2dfMike Marciniszyn ohdr = &qp->s_hdr->u.l.oth; 250f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 251f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 252f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * The lock is needed to synchronize between the sending tasklet, 253f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * the receive interrupt handler, and timeout resends. 254f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 255f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 256f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 257f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Sending responses has higher priority over sending requests. */ 258f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if ((qp->s_flags & QIB_S_RESP_PENDING) && 259f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_make_rc_ack(dev, qp, ohdr, pmtu)) 260f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto done; 261f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 262f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_SEND_OK)) { 263f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(ib_qib_state_ops[qp->state] & QIB_FLUSH_SEND)) 264f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 265f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* We are in the error state, flush the work request. */ 266f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_last == qp->s_head) 267f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 268f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* If DMAs are in progress, we can't flush immediately. */ 269f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (atomic_read(&qp->s_dma_busy)) { 270f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_WAIT_DMA; 271f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 272f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 273f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_last); 27430ab7e230b996f750d4fc24b6bf8214e83effa12Mike Marciniszyn qib_send_complete(qp, wqe, qp->s_last != qp->s_acked ? 27530ab7e230b996f750d4fc24b6bf8214e83effa12Mike Marciniszyn IB_WC_SUCCESS : IB_WC_WR_FLUSH_ERR); 27630ab7e230b996f750d4fc24b6bf8214e83effa12Mike Marciniszyn /* will get called again */ 277f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto done; 278f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 279f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 280f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & (QIB_S_WAIT_RNR | QIB_S_WAIT_ACK)) 281f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 282f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 283f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) { 284f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) { 285f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_WAIT_PSN; 286f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 287f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 288f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sending_psn = qp->s_psn; 289f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sending_hpsn = qp->s_psn - 1; 290f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 291f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 292f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* header size in 32-bit words LRH+BTH = (8+12)/4. */ 293f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords = 5; 294f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 = 0; 295f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 296f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Send a request. */ 297f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_cur); 298f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (qp->s_state) { 299f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: 300f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_NEXT_SEND_OK)) 301f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 302f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 303f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Resend an old request or start a new one. 304f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 305f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We keep track of the current SWQE so that 306f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * we don't reset the "furthest progress" state 307f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * if we need to back up. 308f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 309f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell newreq = 0; 310f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_cur == qp->s_tail) { 311f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check if send work queue is empty. */ 312f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_tail == qp->s_head) 313f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 314f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 315f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If a fence is requested, wait for previous 316f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * RDMA read and atomic operations to finish. 317f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 318f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if ((wqe->wr.send_flags & IB_SEND_FENCE) && 319f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_num_rd_atomic) { 320f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_WAIT_FENCE; 321f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 322f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 323f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->psn = qp->s_next_psn; 324f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell newreq = 1; 325f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 326f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 327f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Note that we have to be careful not to modify the 328f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * original work request since we may need to resend 329f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * it. 330f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 331f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = wqe->length; 332f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ss = &qp->s_sge; 333f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 = qp->s_psn & QIB_PSN_MASK; 334f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (wqe->wr.opcode) { 335f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_SEND: 336f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_SEND_WITH_IMM: 337f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* If no credit, return. */ 338f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(qp->s_flags & QIB_S_UNLIMITED_CREDIT) && 339f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) { 340f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_WAIT_SSN_CREDIT; 341f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 342f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 343f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->lpsn = wqe->psn; 344f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len > pmtu) { 345f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->lpsn += (len - 1) / pmtu; 346f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_FIRST); 347f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = pmtu; 348f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 349f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 350f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_SEND) 351f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_ONLY); 352f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else { 353f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE); 354f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Immediate data comes after the BTH */ 355f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.imm_data = wqe->wr.ex.imm_data; 356f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += 1; 357f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 358f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.send_flags & IB_SEND_SOLICITED) 359f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 |= IB_BTH_SOLICITED; 360f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 |= IB_BTH_REQ_ACK; 361f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_cur == qp->s_size) 362f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = 0; 363f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 364f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 365f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_RDMA_WRITE: 366f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (newreq && !(qp->s_flags & QIB_S_UNLIMITED_CREDIT)) 367f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_lsn++; 368f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 369f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_RDMA_WRITE_WITH_IMM: 370f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* If no credit, return. */ 371f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(qp->s_flags & QIB_S_UNLIMITED_CREDIT) && 372f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) { 373f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_WAIT_SSN_CREDIT; 374f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 375f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 376f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.reth.vaddr = 377f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be64(wqe->wr.wr.rdma.remote_addr); 378f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.reth.rkey = 379f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be32(wqe->wr.wr.rdma.rkey); 380f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.reth.length = cpu_to_be32(len); 381f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += sizeof(struct ib_reth) / sizeof(u32); 382f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->lpsn = wqe->psn; 383f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len > pmtu) { 384f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->lpsn += (len - 1) / pmtu; 385f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_WRITE_FIRST); 386f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = pmtu; 387f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 388f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 389f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_RDMA_WRITE) 390f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_WRITE_ONLY); 391f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else { 392f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = 393f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE); 394f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Immediate data comes after RETH */ 395f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.imm_data = wqe->wr.ex.imm_data; 396f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += 1; 397f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.send_flags & IB_SEND_SOLICITED) 398f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 |= IB_BTH_SOLICITED; 399f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 400f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 |= IB_BTH_REQ_ACK; 401f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_cur == qp->s_size) 402f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = 0; 403f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 404f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 405f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_RDMA_READ: 406f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 407f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Don't allow more operations to be started 408f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * than the QP limits allow. 409f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 410f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (newreq) { 411f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_num_rd_atomic >= 412f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_max_rd_atomic) { 413f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_WAIT_RDMAR; 414f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 415f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 416f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_num_rd_atomic++; 417f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(qp->s_flags & QIB_S_UNLIMITED_CREDIT)) 418f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_lsn++; 419f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 420f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Adjust s_next_psn to count the 421f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * expected number of responses. 422f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 423f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len > pmtu) 424f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_next_psn += (len - 1) / pmtu; 425f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->lpsn = qp->s_next_psn++; 426f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 427f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.reth.vaddr = 428f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be64(wqe->wr.wr.rdma.remote_addr); 429f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.reth.rkey = 430f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be32(wqe->wr.wr.rdma.rkey); 431f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.reth.length = cpu_to_be32(len); 432f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_READ_REQUEST); 433f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32); 434f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ss = NULL; 435f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = 0; 436f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 |= IB_BTH_REQ_ACK; 437f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_cur == qp->s_size) 438f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = 0; 439f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 440f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 441f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_ATOMIC_CMP_AND_SWP: 442f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_ATOMIC_FETCH_AND_ADD: 443f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 444f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Don't allow more operations to be started 445f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * than the QP limits allow. 446f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 447f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (newreq) { 448f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_num_rd_atomic >= 449f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_max_rd_atomic) { 450f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_WAIT_RDMAR; 451f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 452f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 453f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_num_rd_atomic++; 454f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(qp->s_flags & QIB_S_UNLIMITED_CREDIT)) 455f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_lsn++; 456f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->lpsn = wqe->psn; 457f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 458f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) { 459f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(COMPARE_SWAP); 460f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.atomic_eth.swap_data = cpu_to_be64( 461f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.wr.atomic.swap); 462f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.atomic_eth.compare_data = cpu_to_be64( 463f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.wr.atomic.compare_add); 464f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else { 465f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(FETCH_ADD); 466f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.atomic_eth.swap_data = cpu_to_be64( 467f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.wr.atomic.compare_add); 468f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.atomic_eth.compare_data = 0; 469f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 470f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.atomic_eth.vaddr[0] = cpu_to_be32( 471f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.wr.atomic.remote_addr >> 32); 472f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.atomic_eth.vaddr[1] = cpu_to_be32( 473f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.wr.atomic.remote_addr); 474f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.atomic_eth.rkey = cpu_to_be32( 475f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.wr.atomic.rkey); 476f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += sizeof(struct ib_atomic_eth) / sizeof(u32); 477f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ss = NULL; 478f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = 0; 479f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 |= IB_BTH_REQ_ACK; 480f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_cur == qp->s_size) 481f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = 0; 482f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 483f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 484f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: 485f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 486f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 487f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sge.sge = wqe->sg_list[0]; 488f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sge.sg_list = wqe->sg_list + 1; 489f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sge.num_sge = wqe->wr.num_sge; 490f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sge.total_len = wqe->length; 491f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_len = wqe->length; 492f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (newreq) { 493f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_tail++; 494f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_tail >= qp->s_size) 495f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_tail = 0; 496f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 497f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_RDMA_READ) 498f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_psn = wqe->lpsn + 1; 499f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else { 500f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_psn++; 501f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(qp->s_psn, qp->s_next_psn) > 0) 502f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_next_psn = qp->s_psn; 503f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 504f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 505f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 506f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_FIRST): 507f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 508f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * qp->s_state is normally set to the opcode of the 509f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * last packet constructed for new requests and therefore 510f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * is never set to RDMA read response. 511f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * RDMA_READ_RESPONSE_FIRST is used by the ACK processing 512f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * thread to indicate a SEND needs to be restarted from an 513f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * earlier PSN without interferring with the sending thread. 514f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * See qib_restart_rc(). 515f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 516f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); 517f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 518f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_FIRST): 519f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_MIDDLE); 520f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 521f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_MIDDLE): 522f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 = qp->s_psn++ & QIB_PSN_MASK; 523f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(qp->s_psn, qp->s_next_psn) > 0) 524f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_next_psn = qp->s_psn; 525f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ss = &qp->s_sge; 526f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = qp->s_len; 527f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len > pmtu) { 528f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = pmtu; 529f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 530f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 531f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_SEND) 532f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_LAST); 533f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else { 534f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); 535f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Immediate data comes after the BTH */ 536f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.imm_data = wqe->wr.ex.imm_data; 537f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += 1; 538f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 539f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.send_flags & IB_SEND_SOLICITED) 540f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 |= IB_BTH_SOLICITED; 541f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 |= IB_BTH_REQ_ACK; 542f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur++; 543f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_cur >= qp->s_size) 544f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = 0; 545f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 546f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 547f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_LAST): 548f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 549f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * qp->s_state is normally set to the opcode of the 550f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * last packet constructed for new requests and therefore 551f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * is never set to RDMA read response. 552f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * RDMA_READ_RESPONSE_LAST is used by the ACK processing 553f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * thread to indicate a RDMA write needs to be restarted from 554f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * an earlier PSN without interferring with the sending thread. 555f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * See qib_restart_rc(). 556f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 557f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); 558f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 559f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_FIRST): 560f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_WRITE_MIDDLE); 561f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 562f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_MIDDLE): 563f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 = qp->s_psn++ & QIB_PSN_MASK; 564f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(qp->s_psn, qp->s_next_psn) > 0) 565f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_next_psn = qp->s_psn; 566f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ss = &qp->s_sge; 567f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = qp->s_len; 568f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len > pmtu) { 569f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = pmtu; 570f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 571f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 572f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_RDMA_WRITE) 573f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_WRITE_LAST); 574f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else { 575f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); 576f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Immediate data comes after the BTH */ 577f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.imm_data = wqe->wr.ex.imm_data; 578f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += 1; 579f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.send_flags & IB_SEND_SOLICITED) 580f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 |= IB_BTH_SOLICITED; 581f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 582f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 |= IB_BTH_REQ_ACK; 583f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur++; 584f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_cur >= qp->s_size) 585f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = 0; 586f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 587f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 588f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_MIDDLE): 589f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 590f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * qp->s_state is normally set to the opcode of the 591f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * last packet constructed for new requests and therefore 592f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * is never set to RDMA read response. 593f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * RDMA_READ_RESPONSE_MIDDLE is used by the ACK processing 594f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * thread to indicate a RDMA read needs to be restarted from 595f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * an earlier PSN without interferring with the sending thread. 596f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * See qib_restart_rc(). 597f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 598f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = ((qp->s_psn - wqe->psn) & QIB_PSN_MASK) * pmtu; 599f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.reth.vaddr = 600f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be64(wqe->wr.wr.rdma.remote_addr + len); 601f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.reth.rkey = 602f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be32(wqe->wr.wr.rdma.rkey); 603f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.rc.reth.length = cpu_to_be32(wqe->length - len); 604f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_READ_REQUEST); 605f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32); 606f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 = (qp->s_psn & QIB_PSN_MASK) | IB_BTH_REQ_ACK; 607f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_psn = wqe->lpsn + 1; 608f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ss = NULL; 609f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = 0; 610f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur++; 611f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_cur == qp->s_size) 612f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = 0; 613f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 614f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 615f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sending_hpsn = bth2; 616f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell delta = (((int) bth2 - (int) wqe->psn) << 8) >> 8; 617f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (delta && delta % QIB_PSN_CREDIT == 0) 618f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 |= IB_BTH_REQ_ACK; 619f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & QIB_S_SEND_ONE) { 620f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~QIB_S_SEND_ONE; 621f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_WAIT_ACK; 622f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth2 |= IB_BTH_REQ_ACK; 623f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 624f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_len -= len; 625f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_hdrwords = hwords; 626f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur_sge = ss; 627f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur_size = len; 628f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), bth2); 629f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbelldone: 630f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ret = 1; 631f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto unlock; 632f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 633f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellbail: 634f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~QIB_S_BUSY; 635f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellunlock: 636f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 637f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return ret; 638f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 639f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 640f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/** 641f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * qib_send_rc_ack - Construct an ACK packet and send it 642f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @qp: a pointer to the QP 643f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 644f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This is called from qib_rc_rcv() and qib_kreceive(). 645f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Note that RDMA reads and atomics are handled in the 646f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * send side QP state and tasklet. 647f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 648f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellvoid qib_send_rc_ack(struct qib_qp *qp) 649f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 650f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device); 651f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); 652f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_pportdata *ppd = ppd_from_ibp(ibp); 653f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u64 pbc; 654f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u16 lrh0; 655f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 bth0; 656f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 hwords; 657f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 pbufn; 658f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 __iomem *piobuf; 659f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ib_header hdr; 660f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_other_headers *ohdr; 661f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 control; 662f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned long flags; 663f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 664f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 665f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 666f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) 667f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto unlock; 668f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 669f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Don't send ACK or NAK if a RDMA read or atomic is pending. */ 670f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if ((qp->s_flags & QIB_S_RESP_PENDING) || qp->s_rdma_ack_cnt) 671f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto queue_ack; 672f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 673f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Construct the header with s_lock held so APM doesn't change it. */ 674f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr = &hdr.u.oth; 675f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell lrh0 = QIB_LRH_BTH; 676f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* header size in 32-bit words LRH+BTH+AETH = (8+12+4)/4. */ 677f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords = 6; 678f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) { 679f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hwords += qib_make_grh(ibp, &hdr.u.l.grh, 680f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell &qp->remote_ah_attr.grh, hwords, 0); 681f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr = &hdr.u.l.oth; 682f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell lrh0 = QIB_LRH_GRH; 683f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 684f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* read pkey_index w/o lock (its atomic) */ 685f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 = qib_get_pkey(ibp, qp->s_pkey_index) | (OP(ACKNOWLEDGE) << 24); 686f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_mig_state == IB_MIG_MIGRATED) 687f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell bth0 |= IB_BTH_MIG_REQ; 688f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->r_nak_state) 689f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.aeth = cpu_to_be32((qp->r_msn & QIB_MSN_MASK) | 690f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (qp->r_nak_state << 691f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell QIB_AETH_CREDIT_SHIFT)); 692f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else 693f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->u.aeth = qib_compute_aeth(qp); 694f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell lrh0 |= ibp->sl_to_vl[qp->remote_ah_attr.sl] << 12 | 695f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->remote_ah_attr.sl << 4; 696f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdr.lrh[0] = cpu_to_be16(lrh0); 697f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid); 698f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdr.lrh[2] = cpu_to_be16(hwords + SIZE_OF_CRC); 699f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdr.lrh[3] = cpu_to_be16(ppd->lid | qp->remote_ah_attr.src_path_bits); 700f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->bth[0] = cpu_to_be32(bth0); 701f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->bth[1] = cpu_to_be32(qp->remote_qpn); 702f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr->bth[2] = cpu_to_be32(qp->r_ack_psn & QIB_PSN_MASK); 703f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 704f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 705f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 706f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Don't try to send ACKs if the link isn't ACTIVE */ 707f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(ppd->lflags & QIBL_LINKACTIVE)) 708f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto done; 709f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 710f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell control = dd->f_setpbc_control(ppd, hwords + SIZE_OF_CRC, 711f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_srate, lrh0 >> 12); 712f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* length is + 1 for the control dword */ 713f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell pbc = ((u64) control << 32) | (hwords + 1); 714f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 715f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell piobuf = dd->f_getsendbuf(ppd, pbc, &pbufn); 716f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!piobuf) { 717f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 718f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We are out of PIO buffers at the moment. 719f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Pass responsibility for sending the ACK to the 720f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * send tasklet so that when a PIO buffer becomes 721f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * available, the ACK is sent ahead of other outgoing 722f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * packets. 723f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 724f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 725f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto queue_ack; 726f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 727f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 728f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 729f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Write the pbc. 730f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We have to flush after the PBC for correctness 731f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * on some cpus or WC buffer can be written out of order. 732f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 733f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell writeq(pbc, piobuf); 734f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 735f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (dd->flags & QIB_PIO_FLUSH_WC) { 736f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 *hdrp = (u32 *) &hdr; 737f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 738f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_flush_wc(); 739f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_pio_copy(piobuf + 2, hdrp, hwords - 1); 740f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_flush_wc(); 741f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell __raw_writel(hdrp[hwords - 1], piobuf + hwords + 1); 742f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else 743f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_pio_copy(piobuf + 2, (u32 *) &hdr, hwords); 744f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 745f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (dd->flags & QIB_USE_SPCL_TRIG) { 746f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 spcl_off = (pbufn >= dd->piobcnt2k) ? 2047 : 1023; 747f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 748f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_flush_wc(); 749f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell __raw_writel(0xaebecede, piobuf + spcl_off); 750f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 751f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 752f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_flush_wc(); 753f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_sendbuf_done(dd, pbufn); 754f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 7557d7632add8dd99f68b21546efff08a5a162de184Mike Marciniszyn this_cpu_inc(ibp->pmastats->n_unicast_xmit); 756f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto done; 757f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 758f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellqueue_ack: 759f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK) { 760f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rc_qacks++; 761f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_ACK_PENDING | QIB_S_RESP_PENDING; 762f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_nak_state = qp->r_nak_state; 763f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_psn = qp->r_ack_psn; 764f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 765f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Schedule the send tasklet. */ 766f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 767f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 768f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellunlock: 769f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 770f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbelldone: 771f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return; 772f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 773f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 774f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/** 775f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * reset_psn - reset the QP state to send starting from PSN 776f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @qp: the QP 777f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @psn: the packet sequence number to restart at 778f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 779f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This is called from qib_rc_rcv() to process an incoming RC ACK 780f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * for the given QP. 781f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Called at interrupt level with the QP s_lock held. 782f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 783f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic void reset_psn(struct qib_qp *qp, u32 psn) 784f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 785f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 n = qp->s_acked; 786f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_swqe *wqe = get_swqe_ptr(qp, n); 787f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 opcode; 788f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 789f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = n; 790f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 791f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 792f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If we are starting the request from the beginning, 793f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * let the normal send code handle initialization. 794f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 795f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(psn, wqe->psn) <= 0) { 796f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_LAST); 797f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto done; 798f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 799f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 800f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Find the work request opcode corresponding to the given PSN. */ 801f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode = wqe->wr.opcode; 802f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell for (;;) { 803f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int diff; 804f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 805f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++n == qp->s_size) 806f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell n = 0; 807f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (n == qp->s_tail) 808f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 809f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, n); 810f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell diff = qib_cmp24(psn, wqe->psn); 811f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (diff < 0) 812f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 813f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = n; 814f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 815f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If we are starting the request from the beginning, 816f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * let the normal send code handle initialization. 817f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 818f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (diff == 0) { 819f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_LAST); 820f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto done; 821f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 822f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode = wqe->wr.opcode; 823f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 824f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 825f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 826f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Set the state to restart in the middle of a request. 827f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Don't change the s_sge, s_cur_sge, or s_cur_size. 828f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * See qib_make_rc_req(). 829f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 830f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (opcode) { 831f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_SEND: 832f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_SEND_WITH_IMM: 833f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_READ_RESPONSE_FIRST); 834f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 835f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 836f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_RDMA_WRITE: 837f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_RDMA_WRITE_WITH_IMM: 838f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_READ_RESPONSE_LAST); 839f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 840f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 841f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case IB_WR_RDMA_READ: 842f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE); 843f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 844f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 845f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: 846f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 847f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This case shouldn't happen since its only 848f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * one PSN per req. 849f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 850f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_LAST); 851f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 852f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbelldone: 853f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_psn = psn; 854f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 855f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Set QIB_S_WAIT_PSN as qib_rc_complete() may start the timer 856f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * asynchronously before the send tasklet can get scheduled. 857f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Doing it in qib_make_rc_req() is too late. 858f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 859f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if ((qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) && 860f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) 861f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_WAIT_PSN; 862f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 863f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 864f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/* 865f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Back up requester to resend the last un-ACKed request. 866a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell * The QP r_lock and s_lock should be held and interrupts disabled. 867f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 868f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic void qib_restart_rc(struct qib_qp *qp, u32 psn, int wait) 869f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 870f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_swqe *wqe = get_swqe_ptr(qp, qp->s_acked); 871f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ibport *ibp; 872f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 873f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_retry == 0) { 874f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_mig_state == IB_MIG_ARMED) { 875f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_migrate_qp(qp); 876f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_retry = qp->s_retry_cnt; 877f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else if (qp->s_last == qp->s_acked) { 878f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR); 879f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_error_qp(qp, IB_WC_WR_FLUSH_ERR); 880f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return; 881f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else /* XXX need to handle delayed completion */ 882f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return; 883f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else 884f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_retry--; 885f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 886f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp = to_iport(qp->ibqp.device, qp->port_num); 887f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_RDMA_READ) 888f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rc_resends++; 889f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else 890f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK; 891f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 892f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~(QIB_S_WAIT_FENCE | QIB_S_WAIT_RDMAR | 893f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell QIB_S_WAIT_SSN_CREDIT | QIB_S_WAIT_PSN | 894f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell QIB_S_WAIT_ACK); 895f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wait) 896f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_SEND_ONE; 897f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell reset_psn(qp, psn); 898f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 899f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 900f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/* 901f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This is called from s_timer for missing responses. 902f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 903f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic void rc_timeout(unsigned long arg) 904f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 905f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_qp *qp = (struct qib_qp *)arg; 906f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ibport *ibp; 907f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned long flags; 908f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 909a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell spin_lock_irqsave(&qp->r_lock, flags); 910a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell spin_lock(&qp->s_lock); 911f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & QIB_S_TIMER) { 912f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp = to_iport(qp->ibqp.device, qp->port_num); 913f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rc_timeouts++; 914f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~QIB_S_TIMER; 915f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell del_timer(&qp->s_timer); 916f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_restart_rc(qp, qp->s_last_psn + 1, 1); 917f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 918f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 919a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell spin_unlock(&qp->s_lock); 920a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell spin_unlock_irqrestore(&qp->r_lock, flags); 921f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 922f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 923f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/* 924f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This is called from s_timer for RNR timeouts. 925f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 926f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellvoid qib_rc_rnr_retry(unsigned long arg) 927f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 928f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_qp *qp = (struct qib_qp *)arg; 929f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned long flags; 930f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 931f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 932f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & QIB_S_WAIT_RNR) { 933f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~QIB_S_WAIT_RNR; 934f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell del_timer(&qp->s_timer); 935f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 936f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 937f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 938f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 939f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 940f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/* 941f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Set qp->s_sending_psn to the next PSN after the given one. 942f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This would be psn+1 except when RDMA reads are present. 943f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 944f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic void reset_sending_psn(struct qib_qp *qp, u32 psn) 945f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 946f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_swqe *wqe; 947f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 n = qp->s_last; 948f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 949f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Find the work request corresponding to the given PSN. */ 950f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell for (;;) { 951f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, n); 952f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(psn, wqe->lpsn) <= 0) { 953f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_RDMA_READ) 954f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sending_psn = wqe->lpsn + 1; 955f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else 956f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sending_psn = psn + 1; 957f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 958f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 959f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++n == qp->s_size) 960f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell n = 0; 961f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (n == qp->s_tail) 962f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 963f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 964f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 965f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 966f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/* 967f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This should be called with the QP s_lock held and interrupts disabled. 968f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 969f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellvoid qib_rc_send_complete(struct qib_qp *qp, struct qib_ib_header *hdr) 970f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 971f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_other_headers *ohdr; 972f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_swqe *wqe; 973f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct ib_wc wc; 974f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned i; 975f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 opcode; 976f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 psn; 977f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 978f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_OR_FLUSH_SEND)) 979f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return; 980f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 981f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Find out where the BTH is */ 982f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if ((be16_to_cpu(hdr->lrh[0]) & 3) == QIB_LRH_BTH) 983f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr = &hdr->u.oth; 984f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else 985f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr = &hdr->u.l.oth; 986f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 987f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode = be32_to_cpu(ohdr->bth[0]) >> 24; 988f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) && 989f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode <= OP(ATOMIC_ACKNOWLEDGE)) { 990f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell WARN_ON(!qp->s_rdma_ack_cnt); 991f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_rdma_ack_cnt--; 992f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return; 993f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 994f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 995f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell psn = be32_to_cpu(ohdr->bth[2]); 996f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell reset_sending_psn(qp, psn); 997f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 998f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 999f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Start timer after a packet requesting an ACK has been sent and 1000f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * there are still requests that haven't been acked. 1001f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1002f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if ((psn & IB_BTH_REQ_ACK) && qp->s_acked != qp->s_tail && 1003c0af2c057d7ce3f0b260f9380d187a82bb5cab28Mike Marciniszyn !(qp->s_flags & (QIB_S_TIMER | QIB_S_WAIT_RNR | QIB_S_WAIT_PSN)) && 1004c0af2c057d7ce3f0b260f9380d187a82bb5cab28Mike Marciniszyn (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) 1005f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell start_timer(qp); 1006f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1007f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell while (qp->s_last != qp->s_acked) { 1008f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_last); 1009f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 && 1010f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) 1011f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1012f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell for (i = 0; i < wqe->wr.num_sge; i++) { 1013f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_sge *sge = &wqe->sg_list[i]; 1014f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 10156a82649f217023863d6b1740017e6c3dd6685327Mike Marciniszyn qib_put_mr(sge->mr); 1016f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1017f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Post a send completion queue entry if requested. */ 1018f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(qp->s_flags & QIB_S_SIGNAL_REQ_WR) || 1019f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (wqe->wr.send_flags & IB_SEND_SIGNALED)) { 1020f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell memset(&wc, 0, sizeof wc); 1021f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.wr_id = wqe->wr.wr_id; 1022f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.status = IB_WC_SUCCESS; 1023f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.opcode = ib_qib_wc_opcode[wqe->wr.opcode]; 1024f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.byte_len = wqe->length; 1025f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.qp = &qp->ibqp; 1026f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 0); 1027f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1028f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_last >= qp->s_size) 1029f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_last = 0; 1030f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1031f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1032f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If we were waiting for sends to complete before resending, 1033f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * and they are now complete, restart sending. 1034f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1035f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & QIB_S_WAIT_PSN && 1036f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { 1037f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~QIB_S_WAIT_PSN; 1038f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sending_psn = qp->s_psn; 1039f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_sending_hpsn = qp->s_psn - 1; 1040f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 1041f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1042f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 1043f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1044f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic inline void update_last_psn(struct qib_qp *qp, u32 psn) 1045f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 1046f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_last_psn = psn; 1047f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 1048f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1049f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/* 1050f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Generate a SWQE completion. 1051f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This is similar to qib_send_complete but has to check to be sure 1052f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * that the SGEs are not being referenced if the SWQE is being resent. 1053f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1054f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic struct qib_swqe *do_rc_completion(struct qib_qp *qp, 1055f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_swqe *wqe, 1056f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ibport *ibp) 1057f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 1058f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct ib_wc wc; 1059f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned i; 1060f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1061f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1062f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Don't decrement refcount and don't generate a 1063f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * completion if the SWQE is being resent until the send 1064f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * is finished. 1065f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1066f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 || 1067f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { 1068f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell for (i = 0; i < wqe->wr.num_sge; i++) { 1069f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_sge *sge = &wqe->sg_list[i]; 1070f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 10716a82649f217023863d6b1740017e6c3dd6685327Mike Marciniszyn qib_put_mr(sge->mr); 1072f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1073f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Post a send completion queue entry if requested. */ 1074f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(qp->s_flags & QIB_S_SIGNAL_REQ_WR) || 1075f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (wqe->wr.send_flags & IB_SEND_SIGNALED)) { 1076f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell memset(&wc, 0, sizeof wc); 1077f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.wr_id = wqe->wr.wr_id; 1078f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.status = IB_WC_SUCCESS; 1079f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.opcode = ib_qib_wc_opcode[wqe->wr.opcode]; 1080f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.byte_len = wqe->length; 1081f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.qp = &qp->ibqp; 1082f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 0); 1083f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1084f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_last >= qp->s_size) 1085f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_last = 0; 1086f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else 1087f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rc_delayed_comp++; 1088f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1089f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_retry = qp->s_retry_cnt; 1090f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell update_last_psn(qp, wqe->lpsn); 1091f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1092f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1093f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If we are completing a request which is in the process of 1094f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * being resent, we can stop resending it since we know the 1095f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * responder has already seen it. 1096f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1097f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_acked == qp->s_cur) { 1098f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_cur >= qp->s_size) 1099f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_cur = 0; 1100f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_acked = qp->s_cur; 1101f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_cur); 1102f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_acked != qp->s_tail) { 1103f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_LAST); 1104f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_psn = wqe->psn; 1105f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1106f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else { 1107f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (++qp->s_acked >= qp->s_size) 1108f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_acked = 0; 1109f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->state == IB_QPS_SQD && qp->s_acked == qp->s_cur) 1110f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_draining = 0; 1111f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_acked); 1112f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1113f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return wqe; 1114f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 1115f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1116f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/** 1117f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * do_rc_ack - process an incoming RC ACK 1118f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @qp: the QP the ACK came in on 1119f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @psn: the packet sequence number of the ACK 1120f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @opcode: the opcode of the request that resulted in the ACK 1121f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 1122f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This is called from qib_rc_rcv_resp() to process an incoming RC ACK 1123f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * for the given QP. 1124f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Called at interrupt level with the QP s_lock held. 1125f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Returns 1 if OK, 0 if current operation should be aborted (NAK). 1126f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1127f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic int do_rc_ack(struct qib_qp *qp, u32 aeth, u32 psn, int opcode, 1128f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u64 val, struct qib_ctxtdata *rcd) 1129f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 1130f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ibport *ibp; 1131f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell enum ib_wc_status status; 1132f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_swqe *wqe; 1133f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int ret = 0; 1134f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 ack_psn; 1135f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int diff; 1136f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1137f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Remove QP from retry timer */ 1138f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & (QIB_S_TIMER | QIB_S_WAIT_RNR)) { 1139f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~(QIB_S_TIMER | QIB_S_WAIT_RNR); 1140f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell del_timer(&qp->s_timer); 1141f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1142f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1143f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1144f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Note that NAKs implicitly ACK outstanding SEND and RDMA write 1145f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * requests and implicitly NAK RDMA read and atomic requests issued 1146f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * before the NAK'ed request. The MSN won't include the NAK'ed 1147f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * request but will include an ACK'ed request(s). 1148f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1149f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ack_psn = psn; 1150f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (aeth >> 29) 1151f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ack_psn--; 1152f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_acked); 1153f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp = to_iport(qp->ibqp.device, qp->port_num); 1154f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1155f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1156f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * The MSN might be for a later WQE than the PSN indicates so 1157f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * only complete WQEs that the PSN finishes. 1158f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1159f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell while ((diff = qib_cmp24(ack_psn, wqe->lpsn)) >= 0) { 1160f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1161f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * RDMA_READ_RESPONSE_ONLY is a special case since 1162f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * we want to generate completion events for everything 1163f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * before the RDMA read, copy the data, then generate 1164f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * the completion for the read. 1165f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1166f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_RDMA_READ && 1167f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(RDMA_READ_RESPONSE_ONLY) && 1168f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell diff == 0) { 1169f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ret = 1; 1170f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 1171f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1172f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1173f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If this request is a RDMA read or atomic, and the ACK is 1174f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * for a later operation, this ACK NAKs the RDMA read or 1175f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * atomic. In other words, only a RDMA_READ_LAST or ONLY 1176f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * can ACK a RDMA read and likewise for atomic ops. Note 1177f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * that the NAK case can only happen if relaxed ordering is 1178f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * used and requests are sent after an RDMA read or atomic 1179f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * is sent but before the response is received. 1180f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1181f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if ((wqe->wr.opcode == IB_WR_RDMA_READ && 1182f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (opcode != OP(RDMA_READ_RESPONSE_LAST) || diff != 0)) || 1183f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 1184f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) && 1185f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (opcode != OP(ATOMIC_ACKNOWLEDGE) || diff != 0))) { 1186f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Retry this request. */ 1187f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(qp->r_flags & QIB_R_RDMAR_SEQ)) { 1188f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_flags |= QIB_R_RDMAR_SEQ; 1189f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_restart_rc(qp, qp->s_last_psn + 1, 0); 1190f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (list_empty(&qp->rspwait)) { 1191f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_flags |= QIB_R_RSP_SEND; 1192f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_inc(&qp->refcount); 1193f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell list_add_tail(&qp->rspwait, 1194f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell &rcd->qp_wait_list); 1195f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1196f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1197f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1198f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * No need to process the ACK/NAK since we are 1199f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * restarting an earlier request. 1200f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1201f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 1202f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1203f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 1204f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 1205f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u64 *vaddr = wqe->sg_list[0].vaddr; 1206f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell *vaddr = val; 1207f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1208f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_num_rd_atomic && 1209f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (wqe->wr.opcode == IB_WR_RDMA_READ || 1210f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 1211f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) { 1212f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_num_rd_atomic--; 1213f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Restart sending task if fence is complete */ 1214f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if ((qp->s_flags & QIB_S_WAIT_FENCE) && 1215f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell !qp->s_num_rd_atomic) { 1216f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~(QIB_S_WAIT_FENCE | 1217f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell QIB_S_WAIT_ACK); 1218f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 1219f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else if (qp->s_flags & QIB_S_WAIT_RDMAR) { 1220f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~(QIB_S_WAIT_RDMAR | 1221f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell QIB_S_WAIT_ACK); 1222f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 1223f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1224f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1225f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = do_rc_completion(qp, wqe, ibp); 1226f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_acked == qp->s_tail) 1227f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1228f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1229f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1230f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (aeth >> 29) { 1231f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case 0: /* ACK */ 1232f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rc_acks++; 1233f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_acked != qp->s_tail) { 1234f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1235f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We are expecting more ACKs so 1236f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * reset the retransmit timer. 1237f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1238f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell start_timer(qp); 1239f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1240f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We can stop resending the earlier packets and 1241f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * continue with the next packet the receiver wants. 1242f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1243f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(qp->s_psn, psn) <= 0) 1244f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell reset_psn(qp, psn + 1); 1245f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else if (qib_cmp24(qp->s_psn, psn) <= 0) { 1246f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_state = OP(SEND_LAST); 1247f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_psn = psn + 1; 1248f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1249f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & QIB_S_WAIT_ACK) { 1250f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~QIB_S_WAIT_ACK; 1251f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 1252f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1253f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_get_credit(qp, aeth); 1254f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_rnr_retry = qp->s_rnr_retry_cnt; 1255f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_retry = qp->s_retry_cnt; 1256f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell update_last_psn(qp, psn); 1257f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ret = 1; 1258f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 1259f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1260f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case 1: /* RNR NAK */ 1261f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rnr_naks++; 1262f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_acked == qp->s_tail) 1263f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 1264f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & QIB_S_WAIT_RNR) 1265f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 1266f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_rnr_retry == 0) { 1267f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell status = IB_WC_RNR_RETRY_EXC_ERR; 1268f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto class_b; 1269f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1270f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_rnr_retry_cnt < 7) 1271f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_rnr_retry--; 1272f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1273f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* The last valid PSN is the previous PSN. */ 1274f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell update_last_psn(qp, psn - 1); 1275f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1276f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK; 1277f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1278f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell reset_psn(qp, psn); 1279f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1280f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~(QIB_S_WAIT_SSN_CREDIT | QIB_S_WAIT_ACK); 1281f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_WAIT_RNR; 1282f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_timer.function = qib_rc_rnr_retry; 1283f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_timer.expires = jiffies + usecs_to_jiffies( 1284f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ib_qib_rnr_table[(aeth >> QIB_AETH_CREDIT_SHIFT) & 1285f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell QIB_AETH_CREDIT_MASK]); 1286f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell add_timer(&qp->s_timer); 1287f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 1288f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1289f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case 3: /* NAK */ 1290f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_acked == qp->s_tail) 1291f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 1292f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* The last valid PSN is the previous PSN. */ 1293f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell update_last_psn(qp, psn - 1); 1294f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch ((aeth >> QIB_AETH_CREDIT_SHIFT) & 1295f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell QIB_AETH_CREDIT_MASK) { 1296f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case 0: /* PSN sequence error */ 1297f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_seq_naks++; 1298f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1299f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Back up to the responder's expected PSN. 1300f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Note that we might get a NAK in the middle of an 1301f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * RDMA READ response which terminates the RDMA 1302f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * READ. 1303f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1304f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_restart_rc(qp, psn, 0); 1305f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 1306f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1307f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1308f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case 1: /* Invalid Request */ 1309f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell status = IB_WC_REM_INV_REQ_ERR; 1310f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_other_naks++; 1311f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto class_b; 1312f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1313f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case 2: /* Remote Access Error */ 1314f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell status = IB_WC_REM_ACCESS_ERR; 1315f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_other_naks++; 1316f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto class_b; 1317f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1318f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case 3: /* Remote Operation Error */ 1319f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell status = IB_WC_REM_OP_ERR; 1320f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_other_naks++; 1321f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellclass_b: 1322f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_last == qp->s_acked) { 1323f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_send_complete(qp, wqe, status); 1324f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_error_qp(qp, IB_WC_WR_FLUSH_ERR); 1325f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1326f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1327f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1328f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: 1329f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Ignore other reserved NAK error codes */ 1330f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto reserved; 1331f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1332f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_retry = qp->s_retry_cnt; 1333f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_rnr_retry = qp->s_rnr_retry_cnt; 1334f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 1335f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1336f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: /* 2: reserved */ 1337f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellreserved: 1338f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Ignore reserved NAK codes. */ 1339f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 1340f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1341f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1342f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellbail: 1343f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return ret; 1344f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 1345f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1346f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/* 1347f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We have seen an out of sequence RDMA read middle or last packet. 1348f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This ACKs SENDs and RDMA writes up to the first RDMA read or atomic SWQE. 1349f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1350f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic void rdma_seq_err(struct qib_qp *qp, struct qib_ibport *ibp, u32 psn, 1351f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ctxtdata *rcd) 1352f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 1353f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_swqe *wqe; 1354f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1355f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Remove QP from retry timer */ 1356f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & (QIB_S_TIMER | QIB_S_WAIT_RNR)) { 1357f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~(QIB_S_TIMER | QIB_S_WAIT_RNR); 1358f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell del_timer(&qp->s_timer); 1359f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1360f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1361f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_acked); 1362f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1363f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell while (qib_cmp24(psn, wqe->lpsn) > 0) { 1364f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (wqe->wr.opcode == IB_WR_RDMA_READ || 1365f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 1366f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) 1367f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1368f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = do_rc_completion(qp, wqe, ibp); 1369f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1370f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1371f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rdma_seq++; 1372f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_flags |= QIB_R_RDMAR_SEQ; 1373f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_restart_rc(qp, qp->s_last_psn + 1, 0); 1374f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (list_empty(&qp->rspwait)) { 1375f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_flags |= QIB_R_RSP_SEND; 1376f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_inc(&qp->refcount); 1377f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 1378f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1379f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 1380f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1381f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/** 1382f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * qib_rc_rcv_resp - process an incoming RC response packet 1383f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @ibp: the port this packet came in on 1384f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @ohdr: the other headers for this packet 1385f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @data: the packet data 1386f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @tlen: the packet length 1387f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @qp: the QP for this packet 1388f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @opcode: the opcode for this packet 1389f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @psn: the packet sequence number for this packet 1390f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @hdrsize: the header length 1391f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @pmtu: the path MTU 1392f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 1393f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This is called from qib_rc_rcv() to process an incoming RC response 1394f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * packet for the given QP. 1395f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Called at interrupt level. 1396f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1397f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic void qib_rc_rcv_resp(struct qib_ibport *ibp, 1398f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_other_headers *ohdr, 1399f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell void *data, u32 tlen, 1400f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_qp *qp, 1401f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 opcode, 1402f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 psn, u32 hdrsize, u32 pmtu, 1403f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ctxtdata *rcd) 1404f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 1405f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_swqe *wqe; 1406dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn struct qib_pportdata *ppd = ppd_from_ibp(ibp); 1407f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell enum ib_wc_status status; 1408f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned long flags; 1409f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int diff; 1410f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 pad; 1411f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 aeth; 1412f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u64 val; 1413f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1414dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn if (opcode != OP(RDMA_READ_RESPONSE_MIDDLE)) { 1415dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn /* 1416dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn * If ACK'd PSN on SDMA busy list try to make progress to 1417dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn * reclaim SDMA credits. 1418dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn */ 1419dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn if ((qib_cmp24(psn, qp->s_sending_psn) >= 0) && 1420dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) { 1421dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn 1422dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn /* 1423dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn * If send tasklet not running attempt to progress 1424dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn * SDMA queue. 1425dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn */ 1426dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn if (!(qp->s_flags & QIB_S_BUSY)) { 1427dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn /* Acquire SDMA Lock */ 1428dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn spin_lock_irqsave(&ppd->sdma_lock, flags); 1429dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn /* Invoke sdma make progress */ 1430dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn qib_sdma_make_progress(ppd); 1431dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn /* Release SDMA Lock */ 1432dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn spin_unlock_irqrestore(&ppd->sdma_lock, flags); 1433dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn } 1434dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn } 1435dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn } 1436dd04e43d46ad7a4e625a9ff3b270dc0db9abe81dMike Marciniszyn 1437f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 1438414ed90cee32486c50f91b28990443e0dc21c868Mike Marciniszyn if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) 1439414ed90cee32486c50f91b28990443e0dc21c868Mike Marciniszyn goto ack_done; 1440f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1441f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Ignore invalid responses. */ 1442f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(psn, qp->s_next_psn) >= 0) 1443f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_done; 1444f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1445f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Ignore duplicate responses. */ 1446f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell diff = qib_cmp24(psn, qp->s_last_psn); 1447f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(diff <= 0)) { 1448f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Update credits for "ghost" ACKs */ 1449f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (diff == 0 && opcode == OP(ACKNOWLEDGE)) { 1450f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell aeth = be32_to_cpu(ohdr->u.aeth); 1451f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if ((aeth >> 29) == 0) 1452f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_get_credit(qp, aeth); 1453f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1454f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_done; 1455f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1456f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1457f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1458f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Skip everything other than the PSN we expect, if we are waiting 1459f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * for a reply to a restarted RDMA read or atomic op. 1460f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1461f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->r_flags & QIB_R_RDMAR_SEQ) { 1462f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(psn, qp->s_last_psn + 1) != 0) 1463f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_done; 1464f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_flags &= ~QIB_R_RDMAR_SEQ; 1465f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1466f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1467f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(qp->s_acked == qp->s_tail)) 1468f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_done; 1469f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_acked); 1470f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell status = IB_WC_SUCCESS; 1471f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1472f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (opcode) { 1473f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(ACKNOWLEDGE): 1474f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(ATOMIC_ACKNOWLEDGE): 1475f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_FIRST): 1476f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell aeth = be32_to_cpu(ohdr->u.aeth); 1477f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode == OP(ATOMIC_ACKNOWLEDGE)) { 1478f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell __be32 *p = ohdr->u.at.atomic_ack_eth; 1479f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1480f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell val = ((u64) be32_to_cpu(p[0]) << 32) | 1481f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell be32_to_cpu(p[1]); 1482f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else 1483f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell val = 0; 1484f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) || 1485f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode != OP(RDMA_READ_RESPONSE_FIRST)) 1486f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_done; 1487f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdrsize += 4; 1488f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_acked); 1489f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 1490f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_op_err; 1491f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1492f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If this is a response to a resent RDMA read, we 1493f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * have to be careful to copy the data to the right 1494f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * location. 1495f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1496f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, 1497f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe, psn, pmtu); 1498f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto read_middle; 1499f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1500f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_MIDDLE): 1501f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* no AETH, no ACK */ 1502f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1))) 1503f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_seq_err; 1504f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 1505f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_op_err; 1506f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellread_middle: 1507f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(tlen != (hdrsize + pmtu + 4))) 1508f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_len_err; 1509f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(pmtu >= qp->s_rdma_read_len)) 1510f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_len_err; 1511f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1512f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1513f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We got a response so update the timeout. 1514f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 4.096 usec. * (1 << qp->timeout) 1515f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1516f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_TIMER; 1517d0f2faf72d51dacf5c5e8dec7dca22d0395896e2Mike Marciniszyn mod_timer(&qp->s_timer, jiffies + qp->timeout_jiffies); 1518f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_flags & QIB_S_WAIT_ACK) { 1519f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags &= ~QIB_S_WAIT_ACK; 1520f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 1521f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1522f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1523f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode == OP(RDMA_READ_RESPONSE_MIDDLE)) 1524f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_retry = qp->s_retry_cnt; 1525f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1526f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1527f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Update the RDMA receive state but do the copy w/o 1528f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * holding the locks and blocking interrupts. 1529f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1530f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_rdma_read_len -= pmtu; 1531f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell update_last_psn(qp, psn); 1532f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 1533f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_copy_sge(&qp->s_rdma_read_sge, data, pmtu, 0); 1534f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto bail; 1535f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1536f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_ONLY): 1537f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell aeth = be32_to_cpu(ohdr->u.aeth); 1538f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd)) 1539f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_done; 1540f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Get the number of bytes the message was padded by. */ 1541f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 1542f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1543f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Check that the data size is >= 0 && <= pmtu. 1544f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Remember to account for the AETH header (4) and 1545f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * ICRC (4). 1546f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1547f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(tlen < (hdrsize + pad + 8))) 1548f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_len_err; 1549f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1550f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If this is a response to a resent RDMA read, we 1551f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * have to be careful to copy the data to the right 1552f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * location. 1553f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1554f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe = get_swqe_ptr(qp, qp->s_acked); 1555f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, 1556f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wqe, psn, pmtu); 1557f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto read_last; 1558f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1559f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_RESPONSE_LAST): 1560f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* ACKs READ req. */ 1561f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1))) 1562f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_seq_err; 1563f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 1564f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_op_err; 1565f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Get the number of bytes the message was padded by. */ 1566f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 1567f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1568f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Check that the data size is >= 1 && <= pmtu. 1569f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Remember to account for the AETH header (4) and 1570f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * ICRC (4). 1571f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1572f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(tlen <= (hdrsize + pad + 8))) 1573f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_len_err; 1574f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellread_last: 1575f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell tlen -= hdrsize + pad + 8; 1576f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(tlen != qp->s_rdma_read_len)) 1577f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_len_err; 1578f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell aeth = be32_to_cpu(ohdr->u.aeth); 1579f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_copy_sge(&qp->s_rdma_read_sge, data, tlen, 0); 1580f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell WARN_ON(qp->s_rdma_read_sge.num_sge); 1581f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (void) do_rc_ack(qp, aeth, psn, 1582f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell OP(RDMA_READ_RESPONSE_LAST), 0, rcd); 1583f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_done; 1584f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1585f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1586f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellack_op_err: 1587f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell status = IB_WC_LOC_QP_OP_ERR; 1588f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_err; 1589f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1590f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellack_seq_err: 1591f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell rdma_seq_err(qp, ibp, psn, rcd); 1592f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto ack_done; 1593f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1594f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellack_len_err: 1595f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell status = IB_WC_LOC_LEN_ERR; 1596f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellack_err: 1597f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->s_last == qp->s_acked) { 1598f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_send_complete(qp, wqe, status); 1599f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_error_qp(qp, IB_WC_WR_FLUSH_ERR); 1600f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1601f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellack_done: 1602f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 1603f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellbail: 1604f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return; 1605f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 1606f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1607f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/** 1608f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * qib_rc_rcv_error - process an incoming duplicate or error RC packet 1609f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @ohdr: the other headers for this packet 1610f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @data: the packet data 1611f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @qp: the QP for this packet 1612f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @opcode: the opcode for this packet 1613f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @psn: the packet sequence number for this packet 1614f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @diff: the difference between the PSN and the expected PSN 1615f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 1616f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This is called from qib_rc_rcv() to process an unexpected 1617f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * incoming RC packet for the given QP. 1618f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Called at interrupt level. 1619f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Return 1 if no more processing is needed; otherwise return 0 to 1620f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * schedule a response to be sent. 1621f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1622f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic int qib_rc_rcv_error(struct qib_other_headers *ohdr, 1623f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell void *data, 1624f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_qp *qp, 1625f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 opcode, 1626f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 psn, 1627f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int diff, 1628f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ctxtdata *rcd) 1629f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 1630f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); 1631f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ack_entry *e; 1632f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned long flags; 1633f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u8 i, prev; 1634f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int old_req; 1635f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1636f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (diff > 0) { 1637f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1638f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Packet sequence error. 1639f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * A NAK will ACK earlier sends and RDMA writes. 1640f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Don't queue the NAK if we already sent one. 1641f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1642f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!qp->r_nak_state) { 1643f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rc_seqnak++; 1644f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = IB_NAK_PSN_ERROR; 1645f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Use the expected PSN. */ 1646f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_ack_psn = qp->r_psn; 1647f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1648f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Wait to send the sequence NAK until all packets 1649f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * in the receive queue have been processed. 1650f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Otherwise, we end up propagating congestion. 1651f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1652f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (list_empty(&qp->rspwait)) { 1653f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_flags |= QIB_R_RSP_NAK; 1654f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_inc(&qp->refcount); 1655f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 1656f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1657f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1658f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto done; 1659f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1660f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1661f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1662f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Handle a duplicate request. Don't re-execute SEND, RDMA 1663f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * write or atomic op. Don't NAK errors, just silently drop 1664f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * the duplicate request. Note that r_sge, r_len, and 1665f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * r_rcv_len may be in use so don't modify them. 1666f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 1667f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We are supposed to ACK the earliest duplicate PSN but we 1668f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * can coalesce an outstanding duplicate ACK. We have to 1669f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * send the earliest so that RDMA reads can be restarted at 1670f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * the requester's expected PSN. 1671f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 1672f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * First, find where this duplicate PSN falls within the 1673f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * ACKs previously sent. 1674f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * old_req is true if there is an older response that is scheduled 1675f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * to be sent before sending this one. 1676f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1677f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e = NULL; 1678f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell old_req = 1; 1679f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ibp->n_rc_dupreq++; 1680f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1681f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 1682f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1683f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell for (i = qp->r_head_ack_queue; ; i = prev) { 1684f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (i == qp->s_tail_ack_queue) 1685f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell old_req = 0; 1686f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (i) 1687f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell prev = i - 1; 1688f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else 1689f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell prev = QIB_MAX_RDMA_ATOMIC; 1690f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (prev == qp->r_head_ack_queue) { 1691f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e = NULL; 1692f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1693f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1694f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e = &qp->s_ack_queue[prev]; 1695f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!e->opcode) { 1696f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e = NULL; 1697f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1698f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1699f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_cmp24(psn, e->psn) >= 0) { 1700f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (prev == qp->s_tail_ack_queue && 1701f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_cmp24(psn, e->lpsn) <= 0) 1702f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell old_req = 0; 1703f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1704f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1705f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1706f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (opcode) { 1707f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_REQUEST): { 1708f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct ib_reth *reth; 1709f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 offset; 1710f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 len; 1711f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1712f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1713f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If we didn't find the RDMA read request in the ack queue, 1714f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * we can ignore this request. 1715f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1716f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!e || e->opcode != OP(RDMA_READ_REQUEST)) 1717f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto unlock_done; 1718f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* RETH comes after BTH */ 1719f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell reth = &ohdr->u.rc.reth; 1720f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1721f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Address range must be a subset of the original 1722f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * request and start on pmtu boundaries. 1723f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We reuse the old ack_queue slot since the requester 1724f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * should not back up and request an earlier PSN for the 1725f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * same request. 1726f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1727f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell offset = ((psn - e->psn) & QIB_PSN_MASK) * 1728cc6ea1385b43487f6ef03bdc91416c8366d28311Mike Marciniszyn qp->pmtu; 1729f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = be32_to_cpu(reth->length); 1730f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(offset + len != e->rdma_sge.sge_length)) 1731f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto unlock_done; 1732f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (e->rdma_sge.mr) { 17336a82649f217023863d6b1740017e6c3dd6685327Mike Marciniszyn qib_put_mr(e->rdma_sge.mr); 1734f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.mr = NULL; 1735f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1736f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len != 0) { 1737f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 rkey = be32_to_cpu(reth->rkey); 1738f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u64 vaddr = be64_to_cpu(reth->vaddr); 1739f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int ok; 1740f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1741f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ok = qib_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey, 1742f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell IB_ACCESS_REMOTE_READ); 1743f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(!ok)) 1744f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto unlock_done; 1745f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else { 1746f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.vaddr = NULL; 1747f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.length = 0; 1748f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.sge_length = 0; 1749f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1750f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->psn = psn; 1751f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (old_req) 1752f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto unlock_done; 1753f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_tail_ack_queue = prev; 1754f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1755f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1756f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1757f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(COMPARE_SWAP): 1758f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(FETCH_ADD): { 1759f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1760f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * If we didn't find the atomic request in the ack queue 1761f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * or the send tasklet is already backed up to send an 1762f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * earlier entry, we can ignore this request. 1763f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1764f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!e || e->opcode != (u8) opcode || old_req) 1765f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto unlock_done; 1766f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_tail_ack_queue = prev; 1767f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1768f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1769f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1770f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: 1771f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1772f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Ignore this operation if it doesn't request an ACK 1773f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * or an earlier RDMA read or atomic is going to be resent. 1774f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1775f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(psn & IB_BTH_REQ_ACK) || old_req) 1776f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto unlock_done; 1777f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1778f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Resend the most recent ACK if this request is 1779f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * after all the previous RDMA reads and atomics. 1780f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1781f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (i == qp->r_head_ack_queue) { 1782f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 1783f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = 0; 1784f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_ack_psn = qp->r_psn - 1; 1785f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto send_ack; 1786f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1787f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1788f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Try to send a simple ACK to work around a Mellanox bug 1789f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * which doesn't accept a RDMA read response or atomic 1790f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * response as an ACK for earlier SENDs or RDMA writes. 1791f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1792f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!(qp->s_flags & QIB_S_RESP_PENDING)) { 1793f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 1794f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = 0; 1795f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_ack_psn = qp->s_ack_queue[i].psn - 1; 1796f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto send_ack; 1797f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1798f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1799f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Resend the RDMA read or atomic op which 1800f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * ACKs this duplicate request. 1801f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1802f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_tail_ack_queue = i; 1803f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1804f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1805f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_state = OP(ACKNOWLEDGE); 1806f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_RESP_PENDING; 1807f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = 0; 1808f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 1809f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1810f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellunlock_done: 1811f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 1812f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbelldone: 1813f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return 1; 1814f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1815f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellsend_ack: 1816f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return 0; 1817f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 1818f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1819f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellvoid qib_rc_error(struct qib_qp *qp, enum ib_wc_status err) 1820f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 1821f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned long flags; 1822f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int lastwqe; 1823f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1824f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 1825f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell lastwqe = qib_error_qp(qp, err); 1826f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 1827f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1828f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (lastwqe) { 1829f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct ib_event ev; 1830f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1831f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ev.device = qp->ibqp.device; 1832f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ev.element.qp = &qp->ibqp; 1833f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ev.event = IB_EVENT_QP_LAST_WQE_REACHED; 1834f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); 1835f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1836f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 1837f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1838f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellstatic inline void qib_update_ack_queue(struct qib_qp *qp, unsigned n) 1839f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 1840f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned next; 1841f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1842f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell next = n + 1; 1843f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (next > QIB_MAX_RDMA_ATOMIC) 1844f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell next = 0; 1845f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_tail_ack_queue = next; 1846f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_ack_state = OP(ACKNOWLEDGE); 1847f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 1848f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1849f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell/** 1850f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * qib_rc_rcv - process an incoming RC packet 1851f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @rcd: the context pointer 1852f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @hdr: the header of this packet 1853f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @has_grh: true if the header has a GRH 1854f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @data: the packet data 1855f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @tlen: the packet length 1856f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * @qp: the QP for this packet 1857f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * 1858f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * This is called from qib_qp_rcv() to process an incoming RC packet 1859f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * for the given QP. 1860f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Called at interrupt level. 1861f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1862f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellvoid qib_rc_rcv(struct qib_ctxtdata *rcd, struct qib_ib_header *hdr, 1863f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int has_grh, void *data, u32 tlen, struct qib_qp *qp) 1864f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell{ 1865f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ibport *ibp = &rcd->ppd->ibport_data; 1866f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_other_headers *ohdr; 1867f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 opcode; 1868f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 hdrsize; 1869f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 psn; 1870f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 pad; 1871f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct ib_wc wc; 1872cc6ea1385b43487f6ef03bdc91416c8366d28311Mike Marciniszyn u32 pmtu = qp->pmtu; 1873f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int diff; 1874f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct ib_reth *reth; 1875f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell unsigned long flags; 1876f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int ret; 1877f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1878f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check for GRH */ 1879f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!has_grh) { 1880f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr = &hdr->u.oth; 1881f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdrsize = 8 + 12; /* LRH + BTH */ 1882f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else { 1883f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ohdr = &hdr->u.l.oth; 1884f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdrsize = 8 + 40 + 12; /* LRH + GRH + BTH */ 1885f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1886f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1887f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode = be32_to_cpu(ohdr->bth[0]); 1888f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_ruc_check_hdr(ibp, hdr, has_grh, qp, opcode)) 18899fd5473deb421eb7e5575a5f9d7e43ca67c04fe9Mike Marciniszyn return; 1890f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1891f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell psn = be32_to_cpu(ohdr->bth[2]); 1892f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode >>= 24; 1893f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1894f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1895f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Process responses (ACKs) before anything else. Note that the 1896f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * packet sequence number will be for something in the send work 1897f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * queue rather than the expected receive packet sequence number. 1898f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * In other words, this QP is the requester. 1899f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1900f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) && 1901f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode <= OP(ATOMIC_ACKNOWLEDGE)) { 1902f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_rc_rcv_resp(ibp, ohdr, data, tlen, qp, opcode, psn, 1903f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdrsize, pmtu, rcd); 1904a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell return; 1905f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1906f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1907f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Compute 24 bits worth of difference. */ 1908f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell diff = qib_cmp24(psn, qp->r_psn); 1909f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(diff)) { 1910f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qib_rc_rcv_error(ohdr, data, qp, opcode, psn, diff, rcd)) 1911a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell return; 1912f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto send_ack; 1913f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1914f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1915f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check for opcode sequence errors. */ 1916f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (qp->r_state) { 1917f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_FIRST): 1918f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_MIDDLE): 1919f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode == OP(SEND_MIDDLE) || 1920f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(SEND_LAST) || 1921f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(SEND_LAST_WITH_IMMEDIATE)) 1922f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1923f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 1924f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1925f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_FIRST): 1926f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_MIDDLE): 1927f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode == OP(RDMA_WRITE_MIDDLE) || 1928f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(RDMA_WRITE_LAST) || 1929f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) 1930f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1931f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 1932f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1933f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: 1934f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode == OP(SEND_MIDDLE) || 1935f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(SEND_LAST) || 1936f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(SEND_LAST_WITH_IMMEDIATE) || 1937f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(RDMA_WRITE_MIDDLE) || 1938f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(RDMA_WRITE_LAST) || 1939f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) 1940f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 1941f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 1942f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Note that it is up to the requester to not send a new 1943f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * RDMA read or atomic operation before receiving an ACK 1944f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * for the previous operation. 1945f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 1946f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1947f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1948f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1949f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->state == IB_QPS_RTR && !(qp->r_flags & QIB_R_COMM_EST)) { 1950f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_flags |= QIB_R_COMM_EST; 1951f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->ibqp.event_handler) { 1952f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct ib_event ev; 1953f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1954f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ev.device = qp->ibqp.device; 1955f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ev.element.qp = &qp->ibqp; 1956f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ev.event = IB_EVENT_COMM_EST; 1957f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); 1958f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1959f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 1960f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1961f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* OK, process the packet. */ 1962f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell switch (opcode) { 1963f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_FIRST): 1964f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ret = qib_get_rwqe(qp, 0); 1965f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (ret < 0) 1966f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_op_err; 1967f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!ret) 1968f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto rnr_nak; 1969f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_rcv_len = 0; 1970f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* FALLTHROUGH */ 1971f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_MIDDLE): 1972f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_MIDDLE): 1973f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellsend_middle: 1974f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check for invalid length PMTU or posted rwqe len. */ 1975f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(tlen != (hdrsize + pmtu + 4))) 1976f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 1977f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_rcv_len += pmtu; 1978f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(qp->r_rcv_len > qp->r_len)) 1979f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 1980f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_copy_sge(&qp->r_sge, data, pmtu, 1); 1981f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 1982f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1983f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE): 1984f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* consume RWQE */ 1985f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ret = qib_get_rwqe(qp, 1); 1986f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (ret < 0) 1987f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_op_err; 1988f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!ret) 1989f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto rnr_nak; 1990f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto send_last_imm; 1991f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 1992f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_ONLY): 1993f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_ONLY_WITH_IMMEDIATE): 1994f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ret = qib_get_rwqe(qp, 0); 1995f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (ret < 0) 1996f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_op_err; 1997f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!ret) 1998f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto rnr_nak; 1999f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_rcv_len = 0; 2000f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode == OP(SEND_ONLY)) 20012fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszyn goto no_immediate_data; 20022fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszyn /* FALLTHROUGH for SEND_ONLY_WITH_IMMEDIATE */ 2003f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_LAST_WITH_IMMEDIATE): 2004f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellsend_last_imm: 2005f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.ex.imm_data = ohdr->u.imm_data; 2006f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdrsize += 4; 2007f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.wc_flags = IB_WC_WITH_IMM; 20082fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszyn goto send_last; 2009f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(SEND_LAST): 2010f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_LAST): 20112fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszynno_immediate_data: 20122fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszyn wc.wc_flags = 0; 20132fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszyn wc.ex.imm_data = 0; 2014f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellsend_last: 2015f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Get the number of bytes the message was padded by. */ 2016f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 2017f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check for invalid length. */ 2018f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* XXX LAST len should be >= 1 */ 2019f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(tlen < (hdrsize + pad + 4))) 2020f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 2021f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Don't count the CRC. */ 2022f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell tlen -= (hdrsize + pad + 4); 2023f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.byte_len = tlen + qp->r_rcv_len; 2024f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(wc.byte_len > qp->r_len)) 2025f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 2026f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_copy_sge(&qp->r_sge, data, tlen, 1); 20276a82649f217023863d6b1740017e6c3dd6685327Mike Marciniszyn qib_put_ss(&qp->r_sge); 2028f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_msn++; 2029f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!test_and_clear_bit(QIB_R_WRID_VALID, &qp->r_aflags)) 2030f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 2031f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.wr_id = qp->r_wr_id; 2032f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.status = IB_WC_SUCCESS; 2033f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE) || 2034f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE)) 2035f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.opcode = IB_WC_RECV_RDMA_WITH_IMM; 2036f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else 2037f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.opcode = IB_WC_RECV; 2038f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.qp = &qp->ibqp; 2039f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.src_qp = qp->remote_qpn; 2040f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.slid = qp->remote_ah_attr.dlid; 2041f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell wc.sl = qp->remote_ah_attr.sl; 20422fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszyn /* zero fields that are N/A */ 20432fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszyn wc.vendor_err = 0; 20442fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszyn wc.pkey_index = 0; 20452fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszyn wc.dlid_path_bits = 0; 20462fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszyn wc.port_num = 0; 2047f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Signal completion event if the solicited bit is set. */ 2048f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 2049f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (ohdr->bth[0] & 2050f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell cpu_to_be32(IB_BTH_SOLICITED)) != 0); 2051f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell break; 2052f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2053f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_FIRST): 2054f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_ONLY): 2055f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): 2056f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) 2057f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 2058f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* consume RWQE */ 2059f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell reth = &ohdr->u.rc.reth; 2060f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell hdrsize += sizeof(*reth); 2061f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_len = be32_to_cpu(reth->length); 2062f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_rcv_len = 0; 2063f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.sg_list = NULL; 2064f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (qp->r_len != 0) { 2065f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 rkey = be32_to_cpu(reth->rkey); 2066f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u64 vaddr = be64_to_cpu(reth->vaddr); 2067f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int ok; 2068f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2069f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check rkey & NAK */ 2070f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ok = qib_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr, 2071f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell rkey, IB_ACCESS_REMOTE_WRITE); 2072f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(!ok)) 2073f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_acc; 2074f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.num_sge = 1; 2075f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else { 2076f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.num_sge = 0; 2077f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.sge.mr = NULL; 2078f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.sge.vaddr = NULL; 2079f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.sge.length = 0; 2080f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.sge.sge_length = 0; 2081f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2082f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (opcode == OP(RDMA_WRITE_FIRST)) 2083f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto send_middle; 2084f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell else if (opcode == OP(RDMA_WRITE_ONLY)) 20852fc109c890f7d9620cfed8d439be071a8b2a8bbdMike Marciniszyn goto no_immediate_data; 2086f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ret = qib_get_rwqe(qp, 1); 2087f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (ret < 0) 2088f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_op_err; 2089f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!ret) 2090f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto rnr_nak; 20915715f5d44b93507693feca6c62290e123e5ef688Jason Gunthorpe wc.ex.imm_data = ohdr->u.rc.imm_data; 20925715f5d44b93507693feca6c62290e123e5ef688Jason Gunthorpe hdrsize += 4; 20935715f5d44b93507693feca6c62290e123e5ef688Jason Gunthorpe wc.wc_flags = IB_WC_WITH_IMM; 20945715f5d44b93507693feca6c62290e123e5ef688Jason Gunthorpe goto send_last; 2095f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2096f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(RDMA_READ_REQUEST): { 2097f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ack_entry *e; 2098f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 len; 2099f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u8 next; 2100f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2101f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) 2102f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 2103f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell next = qp->r_head_ack_queue + 1; 2104f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* s_ack_queue is size QIB_MAX_RDMA_ATOMIC+1 so use > not >= */ 2105f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (next > QIB_MAX_RDMA_ATOMIC) 2106f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell next = 0; 2107f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 2108f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(next == qp->s_tail_ack_queue)) { 2109f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!qp->s_ack_queue[next].sent) 2110f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv_unlck; 2111f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_update_ack_queue(qp, next); 2112f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2113f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e = &qp->s_ack_queue[qp->r_head_ack_queue]; 2114f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (e->opcode == OP(RDMA_READ_REQUEST) && e->rdma_sge.mr) { 21156a82649f217023863d6b1740017e6c3dd6685327Mike Marciniszyn qib_put_mr(e->rdma_sge.mr); 2116f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.mr = NULL; 2117f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2118f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell reth = &ohdr->u.rc.reth; 2119f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell len = be32_to_cpu(reth->length); 2120f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len) { 2121f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 rkey = be32_to_cpu(reth->rkey); 2122f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u64 vaddr = be64_to_cpu(reth->vaddr); 2123f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell int ok; 2124f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2125f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check rkey & NAK */ 2126f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ok = qib_rkey_ok(qp, &e->rdma_sge, len, vaddr, 2127f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell rkey, IB_ACCESS_REMOTE_READ); 2128f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(!ok)) 2129f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_acc_unlck; 2130f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 2131f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * Update the next expected PSN. We add 1 later 2132f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * below, so only add the remainder here. 2133f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 2134f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (len > pmtu) 2135f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_psn += (len - 1) / pmtu; 2136f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } else { 2137f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.mr = NULL; 2138f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.vaddr = NULL; 2139f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.length = 0; 2140f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.sge_length = 0; 2141f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2142f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->opcode = opcode; 2143f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->sent = 0; 2144f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->psn = psn; 2145f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->lpsn = qp->r_psn; 2146f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* 2147f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * We need to increment the MSN here instead of when we 2148f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * finish sending the result since a duplicate request would 2149f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell * increment it more than once. 2150f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell */ 2151f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_msn++; 2152f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_psn++; 2153f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_state = opcode; 2154f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = 0; 2155f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_head_ack_queue = next; 2156f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2157f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Schedule the send tasklet. */ 2158f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_RESP_PENDING; 2159f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 2160f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2161a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell goto sunlock; 2162f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2163f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2164f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(COMPARE_SWAP): 2165f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell case OP(FETCH_ADD): { 2166f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct ib_atomic_eth *ateth; 2167f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell struct qib_ack_entry *e; 2168f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u64 vaddr; 2169f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic64_t *maddr; 2170f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u64 sdata; 2171f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u32 rkey; 2172f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell u8 next; 2173f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2174f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC))) 2175f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 2176f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell next = qp->r_head_ack_queue + 1; 2177f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (next > QIB_MAX_RDMA_ATOMIC) 2178f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell next = 0; 2179f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 2180f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(next == qp->s_tail_ack_queue)) { 2181f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (!qp->s_ack_queue[next].sent) 2182f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv_unlck; 2183f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_update_ack_queue(qp, next); 2184f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2185f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e = &qp->s_ack_queue[qp->r_head_ack_queue]; 2186f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (e->opcode == OP(RDMA_READ_REQUEST) && e->rdma_sge.mr) { 21876a82649f217023863d6b1740017e6c3dd6685327Mike Marciniszyn qib_put_mr(e->rdma_sge.mr); 2188f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->rdma_sge.mr = NULL; 2189f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2190f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell ateth = &ohdr->u.atomic_eth; 2191f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell vaddr = ((u64) be32_to_cpu(ateth->vaddr[0]) << 32) | 2192f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell be32_to_cpu(ateth->vaddr[1]); 2193f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(vaddr & (sizeof(u64) - 1))) 2194f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv_unlck; 2195f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell rkey = be32_to_cpu(ateth->rkey); 2196f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Check rkey & NAK */ 2197f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (unlikely(!qib_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64), 2198f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell vaddr, rkey, 2199f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell IB_ACCESS_REMOTE_ATOMIC))) 2200f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_acc_unlck; 2201f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Perform atomic OP and save result. */ 2202f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell maddr = (atomic64_t *) qp->r_sge.sge.vaddr; 2203f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell sdata = be64_to_cpu(ateth->swap_data); 2204f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->atomic_data = (opcode == OP(FETCH_ADD)) ? 2205f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (u64) atomic64_add_return(sdata, maddr) - sdata : 2206f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr, 2207f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell be64_to_cpu(ateth->compare_data), 2208f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell sdata); 22096a82649f217023863d6b1740017e6c3dd6685327Mike Marciniszyn qib_put_mr(qp->r_sge.sge.mr); 2210f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_sge.num_sge = 0; 2211f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->opcode = opcode; 2212f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->sent = 0; 2213f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->psn = psn; 2214f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell e->lpsn = psn; 2215f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_msn++; 2216f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_psn++; 2217f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_state = opcode; 2218f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = 0; 2219f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_head_ack_queue = next; 2220f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2221f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Schedule the send tasklet. */ 2222f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->s_flags |= QIB_S_RESP_PENDING; 2223f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_schedule_send(qp); 2224f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2225a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell goto sunlock; 2226f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2227f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2228f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell default: 2229f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* NAK unknown opcodes. */ 2230f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto nack_inv; 2231f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2232f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_psn++; 2233f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_state = opcode; 2234f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_ack_psn = psn; 2235f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = 0; 2236f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Send an ACK if requested or required. */ 2237f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (psn & (1 << 31)) 2238f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell goto send_ack; 2239a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell return; 2240f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2241f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellrnr_nak: 2242f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = IB_RNR_NAK | qp->r_min_rnr_timer; 2243f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_ack_psn = qp->r_psn; 2244f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Queue RNR NAK for later */ 2245f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (list_empty(&qp->rspwait)) { 2246f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_flags |= QIB_R_RSP_NAK; 2247f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_inc(&qp->refcount); 2248f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 2249f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2250a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell return; 2251f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2252f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellnack_op_err: 2253f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_rc_error(qp, IB_WC_LOC_QP_OP_ERR); 2254f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = IB_NAK_REMOTE_OPERATIONAL_ERROR; 2255f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_ack_psn = qp->r_psn; 2256f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Queue NAK for later */ 2257f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (list_empty(&qp->rspwait)) { 2258f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_flags |= QIB_R_RSP_NAK; 2259f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_inc(&qp->refcount); 2260f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 2261f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2262a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell return; 2263f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2264f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellnack_inv_unlck: 2265f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 2266f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellnack_inv: 2267f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_rc_error(qp, IB_WC_LOC_QP_OP_ERR); 2268f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = IB_NAK_INVALID_REQUEST; 2269f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_ack_psn = qp->r_psn; 2270f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell /* Queue NAK for later */ 2271f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell if (list_empty(&qp->rspwait)) { 2272f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_flags |= QIB_R_RSP_NAK; 2273f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell atomic_inc(&qp->refcount); 2274f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 2275f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell } 2276a5210c12b7c4e34e904f4820a4abd048a2d75db5Ralph Campbell return; 2277f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2278f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellnack_acc_unlck: 2279f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 2280f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellnack_acc: 2281f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_rc_error(qp, IB_WC_LOC_PROT_ERR); 2282f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; 2283f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qp->r_ack_psn = qp->r_psn; 2284f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellsend_ack: 2285f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell qib_send_rc_ack(qp); 2286f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell return; 2287f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell 2288f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbellsunlock: 2289f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 2290f931551bafe1f10ded7f5282e2aa162c267a2e5dRalph Campbell} 2291