ipath_rc.c revision 6ed89b9574776d4178f1ad754d20e4f1e5a4b6c8
197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/* 2759d57686dab8169ca68bbf938ce8e965d1e107aBryan O'Sullivan * Copyright (c) 2006 QLogic, Inc. All rights reserved. 397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. 497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * This software is available to you under a choice of one of two 697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * licenses. You may choose to be licensed under the terms of the GNU 797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * General Public License (GPL) Version 2, available from the file 897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * COPYING in the main directory of this source tree, or the 997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * OpenIB.org BSD license below: 1097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 1197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Redistribution and use in source and binary forms, with or 1297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * without modification, are permitted provided that the following 1397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * conditions are met: 1497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 1597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * - Redistributions of source code must retain the above 1697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * copyright notice, this list of conditions and the following 1797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * disclaimer. 1897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 1997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * - Redistributions in binary form must reproduce the above 2097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * copyright notice, this list of conditions and the following 2197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * disclaimer in the documentation and/or other materials 2297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * provided with the distribution. 2397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 2497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 2597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 2697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 2797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 2897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 2997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 3097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 3197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * SOFTWARE. 3297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 3397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 3497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan#include "ipath_verbs.h" 3534b2aafea38efdf02cd8107a6e1057e2a297c447Bryan O'Sullivan#include "ipath_kernel.h" 3697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 3797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/* cut down ridiculously long IB macro names */ 3897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan#define OP(x) IB_OPCODE_RC_##x 3997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 403859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbellstatic u32 restart_sge(struct ipath_sge_state *ss, struct ipath_swqe *wqe, 413859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u32 psn, u32 pmtu) 423859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell{ 433859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u32 len; 443859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 453859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = ((psn - wqe->psn) & IPATH_PSN_MASK) * pmtu; 463859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ss->sge = wqe->sg_list[0]; 473859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ss->sg_list = wqe->sg_list + 1; 483859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ss->num_sge = wqe->wr.num_sge; 493859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ipath_skip_sge(ss, len); 503859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell return wqe->length - len; 513859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell} 523859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 5397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 5497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ipath_init_restart- initialize the qp->s_sge after a restart 5597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: the QP who's SGE we're restarting 5697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @wqe: the work queue to initialize the QP's SGE from 5797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 5812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * The QP s_lock should be held and interrupts disabled. 5997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 6097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanstatic void ipath_init_restart(struct ipath_qp *qp, struct ipath_swqe *wqe) 6197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 6297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_ibdev *dev; 6397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 643859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, 653859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ib_mtu_enum_to_int(qp->path_mtu)); 6697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev = to_idev(qp->ibqp.device); 6797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_lock(&dev->pending_lock); 6894b8d9f98d7f535037eb9845b81396f667b4f727Bryan O'Sullivan if (list_empty(&qp->timerwait)) 6997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan list_add_tail(&qp->timerwait, 7097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan &dev->pending[dev->pending_index]); 7197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_unlock(&dev->pending_lock); 7297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 7397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 7497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 7597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ipath_make_rc_ack - construct a response packet (ACK, NAK, or RDMA read) 7697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: a pointer to the QP 7797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @ohdr: a pointer to the IB header being constructed 7897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @pmtu: the path MTU 7997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 803859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Return 1 if constructed; otherwise, return 0. 813859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Note that we are in the responder's side of the QP context. 8297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Note the QP s_lock must be held. 8397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 843859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbellstatic int ipath_make_rc_ack(struct ipath_qp *qp, 853859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell struct ipath_other_headers *ohdr, 863859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u32 pmtu, u32 *bth0p, u32 *bth2p) 8797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 883859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell struct ipath_ack_entry *e; 8997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 hwords; 9097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 len; 9197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 bth0; 923859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u32 bth2; 9397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 9497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* header size in 32-bit words LRH+BTH = (8+12)/4. */ 9597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords = 5; 9697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 9797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan switch (qp->s_ack_state) { 983859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(RDMA_READ_RESPONSE_LAST): 993859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(RDMA_READ_RESPONSE_ONLY): 1003859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(ATOMIC_ACKNOWLEDGE): 1013859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(ACKNOWLEDGE); 1023859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* FALLTHROUGH */ 1033859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(ACKNOWLEDGE): 1043859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* Check for no next entry in the queue. */ 1053859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (qp->r_head_ack_queue == qp->s_tail_ack_queue) { 1063859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (qp->s_flags & IPATH_S_ACK_PENDING) 1073859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto normal; 1083859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 1093859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 1103859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 1113859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e = &qp->s_ack_queue[qp->s_tail_ack_queue]; 1123859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (e->opcode == OP(RDMA_READ_REQUEST)) { 1133859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* Copy SGE state in case we need to resend */ 1143859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_rdma_sge = e->rdma_sge; 1153859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_cur_sge = &qp->s_ack_rdma_sge; 1163859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = e->rdma_sge.sge.sge_length; 1173859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (len > pmtu) { 1183859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = pmtu; 1193859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST); 1203859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } else { 1213859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY); 1223859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (++qp->s_tail_ack_queue > 1233859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell IPATH_MAX_RDMA_ATOMIC) 1243859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_tail_ack_queue = 0; 1253859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 1263859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.aeth = ipath_compute_aeth(qp); 1273859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell hwords++; 1283859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_rdma_psn = e->psn; 1293859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell bth2 = qp->s_ack_rdma_psn++ & IPATH_PSN_MASK; 1303859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } else { 1313859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* COMPARE_SWAP or FETCH_ADD */ 1323859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_cur_sge = NULL; 1333859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = 0; 1343859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); 1353859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.at.aeth = ipath_compute_aeth(qp); 1363859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.at.atomic_ack_eth[0] = 1373859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell cpu_to_be32(e->atomic_data >> 32); 1383859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.at.atomic_ack_eth[1] = 1393859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell cpu_to_be32(e->atomic_data); 1403859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell hwords += sizeof(ohdr->u.at) / sizeof(u32); 1413859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell bth2 = e->psn; 1423859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (++qp->s_tail_ack_queue > IPATH_MAX_RDMA_ATOMIC) 1433859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_tail_ack_queue = 0; 1443859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 14597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth0 = qp->s_ack_state << 24; 14697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 14797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 14897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_FIRST): 14997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE); 15097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 15197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_MIDDLE): 1523859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = qp->s_ack_rdma_sge.sge.sge_length; 15397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (len > pmtu) 15497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = pmtu; 15597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 15697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.aeth = ipath_compute_aeth(qp); 15797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords++; 15897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST); 1593859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (++qp->s_tail_ack_queue > IPATH_MAX_RDMA_ATOMIC) 1603859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_tail_ack_queue = 0; 16197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 16297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth0 = qp->s_ack_state << 24; 1633859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell bth2 = qp->s_ack_rdma_psn++ & IPATH_PSN_MASK; 16497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 16597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 16697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan default: 1673859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell normal: 16812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan /* 1693859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Send a regular ACK. 1703859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Set the s_ack_state so we wait until after sending 1713859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * the ACK before setting s_ack_state to ACKNOWLEDGE 1723859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * (see above). 17312eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan */ 1743859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); 1753859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_flags &= ~IPATH_S_ACK_PENDING; 1763859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_cur_sge = NULL; 17712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan if (qp->s_nak_state) 1783859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.aeth = 1793859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell cpu_to_be32((qp->r_msn & IPATH_MSN_MASK) | 1803859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell (qp->s_nak_state << 1813859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell IPATH_AETH_CREDIT_SHIFT)); 18212eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan else 18312eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan ohdr->u.aeth = ipath_compute_aeth(qp); 18497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords++; 1853859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = 0; 1863859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell bth0 = OP(ACKNOWLEDGE) << 24; 1873859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell bth2 = qp->s_ack_psn & IPATH_PSN_MASK; 18897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 18997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_hdrwords = hwords; 19097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur_size = len; 1913859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell *bth0p = bth0; 1923859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell *bth2p = bth2; 1933859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell return 1; 19497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 19512eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivanbail: 1963859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell return 0; 19797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 19897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 19997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 20097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ipath_make_rc_req - construct a request packet (SEND, RDMA r/w, ATOMIC) 20197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: a pointer to the QP 20297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @ohdr: a pointer to the IB header being constructed 20397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @pmtu: the path MTU 20497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @bth0p: pointer to the BTH opcode word 20597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @bth2p: pointer to the BTH PSN word 20697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 20797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Return 1 if constructed; otherwise, return 0. 20812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * Note the QP s_lock must be held and interrupts disabled. 20997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 210ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivanint ipath_make_rc_req(struct ipath_qp *qp, 211ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan struct ipath_other_headers *ohdr, 212ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan u32 pmtu, u32 *bth0p, u32 *bth2p) 21397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 21497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_ibdev *dev = to_idev(qp->ibqp.device); 21597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_sge_state *ss; 21697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_swqe *wqe; 21797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 hwords; 21897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 len; 21997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 bth0; 22097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 bth2; 22197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan char newreq; 22297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 2233859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* Sending responses has higher priority over sending requests. */ 2243859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if ((qp->r_head_ack_queue != qp->s_tail_ack_queue || 2253859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell (qp->s_flags & IPATH_S_ACK_PENDING) || 2263859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state != IB_OPCODE_RC_ACKNOWLEDGE) && 2273859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ipath_make_rc_ack(qp, ohdr, pmtu, bth0p, bth2p)) 2283859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto done; 2293859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 23097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK) || 231c3af664adbe06803931dbc7a3c8588982d72fac1Ralph Campbell qp->s_rnr_timeout || qp->s_wait_credit) 2323859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 23397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 2346022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan /* Limit the number of packets sent without an ACK. */ 2356022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan if (ipath_cmp24(qp->s_psn, qp->s_last_psn + IPATH_PSN_CREDIT) > 0) { 2366022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan qp->s_wait_credit = 1; 2376022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan dev->n_rc_stalls++; 2383859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 2396022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan } 2406022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan 24197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* header size in 32-bit words LRH+BTH = (8+12)/4. */ 24297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords = 5; 24397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth0 = 0; 24497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 24597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Send a request. */ 24697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe = get_swqe_ptr(qp, qp->s_cur); 24797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan switch (qp->s_state) { 24897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan default: 24997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 25097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Resend an old request or start a new one. 25197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 25297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * We keep track of the current SWQE so that 25397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * we don't reset the "furthest progress" state 25497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * if we need to back up. 25597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 25697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan newreq = 0; 25797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_cur == qp->s_tail) { 25897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check if send work queue is empty. */ 25997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_tail == qp->s_head) 2603859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 2613859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 2623859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * If a fence is requested, wait for previous 2633859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * RDMA read and atomic operations to finish. 2643859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 2653859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if ((wqe->wr.send_flags & IB_SEND_FENCE) && 2663859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_num_rd_atomic) { 2673859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_flags |= IPATH_S_FENCE_PENDING; 2683859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 2693859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 2706022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan wqe->psn = qp->s_next_psn; 27197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan newreq = 1; 27297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 27397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 27497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Note that we have to be careful not to modify the 27597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * original work request since we may need to resend 27697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * it. 27797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 27813b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell len = wqe->length; 27997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ss = &qp->s_sge; 28097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth2 = 0; 28197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan switch (wqe->wr.opcode) { 28297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case IB_WR_SEND: 28397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case IB_WR_SEND_WITH_IMM: 28497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* If no credit, return. */ 28597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_lsn != (u32) -1 && 28697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) 2873859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 28897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe->lpsn = wqe->psn; 28997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (len > pmtu) { 29097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe->lpsn += (len - 1) / pmtu; 29197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(SEND_FIRST); 29297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = pmtu; 29397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 29497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 29597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_SEND) 29697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(SEND_ONLY); 29797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 29897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE); 29997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Immediate data comes after the BTH */ 30097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.imm_data = wqe->wr.imm_data; 30197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords += 1; 30297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 30397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.send_flags & IB_SEND_SOLICITED) 30497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth0 |= 1 << 23; 30597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth2 = 1 << 31; /* Request ACK. */ 30697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (++qp->s_cur == qp->s_size) 30797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur = 0; 30897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 30997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 31097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case IB_WR_RDMA_WRITE: 3116700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (newreq && qp->s_lsn != (u32) -1) 31297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_lsn++; 31397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 31497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case IB_WR_RDMA_WRITE_WITH_IMM: 31597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* If no credit, return. */ 31697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_lsn != (u32) -1 && 31797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) 3183859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 31997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.rc.reth.vaddr = 32097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan cpu_to_be64(wqe->wr.wr.rdma.remote_addr); 32197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.rc.reth.rkey = 32297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan cpu_to_be32(wqe->wr.wr.rdma.rkey); 32397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.rc.reth.length = cpu_to_be32(len); 3243859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell hwords += sizeof(struct ib_reth) / sizeof(u32); 32597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe->lpsn = wqe->psn; 32697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (len > pmtu) { 32797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe->lpsn += (len - 1) / pmtu; 32897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(RDMA_WRITE_FIRST); 32997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = pmtu; 33097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 33197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 33297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_RDMA_WRITE) 33397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(RDMA_WRITE_ONLY); 33497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 33597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = 33697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE); 3376700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan /* Immediate data comes after RETH */ 33897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.rc.imm_data = wqe->wr.imm_data; 33997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords += 1; 34097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.send_flags & IB_SEND_SOLICITED) 34197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth0 |= 1 << 23; 34297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 34397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth2 = 1 << 31; /* Request ACK. */ 34497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (++qp->s_cur == qp->s_size) 34597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur = 0; 34697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 34797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 34897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case IB_WR_RDMA_READ: 3493859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 3503859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Don't allow more operations to be started 3513859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * than the QP limits allow. 3523859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 35397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (newreq) { 3543859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (qp->s_num_rd_atomic >= 3553859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_max_rd_atomic) { 3563859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_flags |= IPATH_S_RDMAR_PENDING; 3573859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 3583859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 3593859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_num_rd_atomic++; 3606700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (qp->s_lsn != (u32) -1) 3616700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_lsn++; 36297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 36397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Adjust s_next_psn to count the 36497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * expected number of responses. 36597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 36697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (len > pmtu) 36797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_next_psn += (len - 1) / pmtu; 36897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe->lpsn = qp->s_next_psn++; 36997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 3703859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.rc.reth.vaddr = 3713859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell cpu_to_be64(wqe->wr.wr.rdma.remote_addr); 3723859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.rc.reth.rkey = 3733859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell cpu_to_be32(wqe->wr.wr.rdma.rkey); 3743859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.rc.reth.length = cpu_to_be32(len); 3753859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_state = OP(RDMA_READ_REQUEST); 3763859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32); 37797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ss = NULL; 37897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = 0; 37997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (++qp->s_cur == qp->s_size) 38097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur = 0; 38197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 38297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 38397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case IB_WR_ATOMIC_CMP_AND_SWP: 38497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case IB_WR_ATOMIC_FETCH_AND_ADD: 3853859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 3863859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Don't allow more operations to be started 3873859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * than the QP limits allow. 3883859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 38997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (newreq) { 3903859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (qp->s_num_rd_atomic >= 3913859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_max_rd_atomic) { 3923859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_flags |= IPATH_S_RDMAR_PENDING; 3933859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 3943859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 3953859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_num_rd_atomic++; 3966700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (qp->s_lsn != (u32) -1) 3976700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_lsn++; 39897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe->lpsn = wqe->psn; 39997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 4003859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) { 4013859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_state = OP(COMPARE_SWAP); 4023859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.atomic_eth.swap_data = cpu_to_be64( 4033859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.wr.atomic.swap); 4043859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.atomic_eth.compare_data = cpu_to_be64( 4053859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.wr.atomic.compare_add); 4063859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } else { 4073859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_state = OP(FETCH_ADD); 4083859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.atomic_eth.swap_data = cpu_to_be64( 4093859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.wr.atomic.compare_add); 4103859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.atomic_eth.compare_data = 0; 4113859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 4123859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.atomic_eth.vaddr[0] = cpu_to_be32( 4133859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.wr.atomic.remote_addr >> 32); 4143859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.atomic_eth.vaddr[1] = cpu_to_be32( 4153859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.wr.atomic.remote_addr); 4163859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.atomic_eth.rkey = cpu_to_be32( 4173859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.wr.atomic.rkey); 4183859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell hwords += sizeof(struct ib_atomic_eth) / sizeof(u32); 41997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ss = NULL; 42097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = 0; 4213859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (++qp->s_cur == qp->s_size) 4223859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_cur = 0; 42397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 42497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 42597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan default: 4263859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 42797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 42813b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell qp->s_sge.sge = wqe->sg_list[0]; 42913b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell qp->s_sge.sg_list = wqe->sg_list + 1; 43013b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell qp->s_sge.num_sge = wqe->wr.num_sge; 43113b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell qp->s_len = wqe->length; 43297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (newreq) { 43397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_tail++; 43497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_tail >= qp->s_size) 43597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_tail = 0; 43697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 43713b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell bth2 |= qp->s_psn & IPATH_PSN_MASK; 43813b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell if (wqe->wr.opcode == IB_WR_RDMA_READ) 43913b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell qp->s_psn = wqe->lpsn + 1; 44013b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell else { 44113b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell qp->s_psn++; 4426f5c407460bba332d6bee52e19f2305539395511Ralph Campbell if (ipath_cmp24(qp->s_psn, qp->s_next_psn) > 0) 44313b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell qp->s_next_psn = qp->s_psn; 44413b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell } 44512eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan /* 44612eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * Put the QP on the pending list so lost ACKs will cause 44712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * a retry. More than one request can be pending so the 44812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * QP may already be on the dev->pending list. 44912eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan */ 45097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_lock(&dev->pending_lock); 45194b8d9f98d7f535037eb9845b81396f667b4f727Bryan O'Sullivan if (list_empty(&qp->timerwait)) 45297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan list_add_tail(&qp->timerwait, 45397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan &dev->pending[dev->pending_index]); 45497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_unlock(&dev->pending_lock); 45597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 45697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 45797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_FIRST): 45897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 45912eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * This case can only happen if a send is restarted. 46012eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * See ipath_restart_rc(). 46197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 46297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_init_restart(qp, wqe); 46397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 46497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_FIRST): 46597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(SEND_MIDDLE); 46697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 46797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_MIDDLE): 46827b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan bth2 = qp->s_psn++ & IPATH_PSN_MASK; 4696f5c407460bba332d6bee52e19f2305539395511Ralph Campbell if (ipath_cmp24(qp->s_psn, qp->s_next_psn) > 0) 47097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_next_psn = qp->s_psn; 47197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ss = &qp->s_sge; 47297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = qp->s_len; 47397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (len > pmtu) { 47497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = pmtu; 47597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 47697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 47797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_SEND) 47897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(SEND_LAST); 47997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 48097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); 48197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Immediate data comes after the BTH */ 48297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.imm_data = wqe->wr.imm_data; 48397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords += 1; 48497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 48597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.send_flags & IB_SEND_SOLICITED) 48697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth0 |= 1 << 23; 48797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth2 |= 1 << 31; /* Request ACK. */ 48897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur++; 48997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_cur >= qp->s_size) 49097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur = 0; 49197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 49297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 49397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_LAST): 49497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 49597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * This case can only happen if a RDMA write is restarted. 49697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * See ipath_restart_rc(). 49797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 49897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_init_restart(qp, wqe); 49997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 50097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_FIRST): 50197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(RDMA_WRITE_MIDDLE); 50297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 50397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_MIDDLE): 50427b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan bth2 = qp->s_psn++ & IPATH_PSN_MASK; 5056f5c407460bba332d6bee52e19f2305539395511Ralph Campbell if (ipath_cmp24(qp->s_psn, qp->s_next_psn) > 0) 50697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_next_psn = qp->s_psn; 50797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ss = &qp->s_sge; 50897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = qp->s_len; 50997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (len > pmtu) { 51097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = pmtu; 51197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 51297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 51397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_RDMA_WRITE) 51497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(RDMA_WRITE_LAST); 51597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 51697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); 51797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Immediate data comes after the BTH */ 51897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.imm_data = wqe->wr.imm_data; 51997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords += 1; 52097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.send_flags & IB_SEND_SOLICITED) 52197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth0 |= 1 << 23; 52297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 52397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth2 |= 1 << 31; /* Request ACK. */ 52497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur++; 52597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_cur >= qp->s_size) 52697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur = 0; 52797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 52897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 52997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_MIDDLE): 53097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 53197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * This case can only happen if a RDMA read is restarted. 53297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * See ipath_restart_rc(). 53397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 53497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_init_restart(qp, wqe); 53527b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan len = ((qp->s_psn - wqe->psn) & IPATH_PSN_MASK) * pmtu; 53697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.rc.reth.vaddr = 53797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan cpu_to_be64(wqe->wr.wr.rdma.remote_addr + len); 53897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.rc.reth.rkey = 53997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan cpu_to_be32(wqe->wr.wr.rdma.rkey); 54097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.rc.reth.length = cpu_to_be32(qp->s_len); 54197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(RDMA_READ_REQUEST); 5423859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32); 54327b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan bth2 = qp->s_psn++ & IPATH_PSN_MASK; 5446f5c407460bba332d6bee52e19f2305539395511Ralph Campbell if (ipath_cmp24(qp->s_psn, qp->s_next_psn) > 0) 54597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_next_psn = qp->s_psn; 54697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ss = NULL; 54797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = 0; 54897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur++; 54997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_cur == qp->s_size) 55097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur = 0; 55197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 55297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 5536022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan if (ipath_cmp24(qp->s_psn, qp->s_last_psn + IPATH_PSN_CREDIT - 1) >= 0) 5546022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan bth2 |= 1 << 31; /* Request ACK. */ 55597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_len -= len; 55697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_hdrwords = hwords; 55797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur_sge = ss; 55897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur_size = len; 55997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan *bth0p = bth0 | (qp->s_state << 24); 56097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan *bth2p = bth2; 5613859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbelldone: 56297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan return 1; 56397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 5643859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbellbail: 56597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan return 0; 56697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 56797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 56897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 569ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * send_rc_ack - Construct an ACK packet and send it 570ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * @qp: a pointer to the QP 57197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 572ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * This is called from ipath_rc_rcv() and only uses the receive 573ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * side QP state. 5743859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Note that RDMA reads and atomics are handled in the 5753859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * send side QP state and tasklet. 57697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 57797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanstatic void send_rc_ack(struct ipath_qp *qp) 57897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 57997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_ibdev *dev = to_idev(qp->ibqp.device); 58097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u16 lrh0; 58197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 bth0; 582ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan u32 hwords; 583ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan struct ipath_ib_header hdr; 58497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_other_headers *ohdr; 58535ff032e65ab5cc03bbba46cefece7376c7c562fRalph Campbell unsigned long flags; 58697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 5873859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* Don't send ACK or NAK if a RDMA read or atomic is pending. */ 5883859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (qp->r_head_ack_queue != qp->s_tail_ack_queue) 5893859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto queue_ack; 5903859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 59197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Construct the header. */ 592ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan ohdr = &hdr.u.oth; 59327b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan lrh0 = IPATH_LRH_BTH; 59497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* header size in 32-bit words LRH+BTH+AETH = (8+12+4)/4. */ 595ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hwords = 6; 59697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) { 597ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hwords += ipath_make_grh(dev, &hdr.u.l.grh, 598ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan &qp->remote_ah_attr.grh, 599ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hwords, 0); 600ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan ohdr = &hdr.u.l.oth; 60127b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan lrh0 = IPATH_LRH_GRH; 60297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 60312eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan /* read pkey_index w/o lock (its atomic) */ 6043859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell bth0 = ipath_get_pkey(dev->dd, qp->s_pkey_index) | 6053859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell OP(ACKNOWLEDGE) << 24; 60612eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan if (qp->r_nak_state) 60727b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan ohdr->u.aeth = cpu_to_be32((qp->r_msn & IPATH_MSN_MASK) | 60812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan (qp->r_nak_state << 60927b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan IPATH_AETH_CREDIT_SHIFT)); 61012eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan else 61112eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan ohdr->u.aeth = ipath_compute_aeth(qp); 61297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan lrh0 |= qp->remote_ah_attr.sl << 4; 613ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hdr.lrh[0] = cpu_to_be16(lrh0); 614ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid); 615ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hdr.lrh[2] = cpu_to_be16(hwords + SIZE_OF_CRC); 61634b2aafea38efdf02cd8107a6e1057e2a297c447Bryan O'Sullivan hdr.lrh[3] = cpu_to_be16(dev->dd->ipath_lid); 61797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->bth[0] = cpu_to_be32(bth0); 61897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->bth[1] = cpu_to_be32(qp->remote_qpn); 61927b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan ohdr->bth[2] = cpu_to_be32(qp->r_ack_psn & IPATH_PSN_MASK); 62097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 62197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 62297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * If we can send the ACK, clear the ACK state. 62397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 624ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan if (ipath_verbs_send(dev->dd, hwords, (u32 *) &hdr, 0, NULL) == 0) { 62597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_unicast_xmit++; 6263859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto done; 62797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 6283859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 6293859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 6303859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * We are out of PIO buffers at the moment. 6313859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Pass responsibility for sending the ACK to the 6323859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * send tasklet so that when a PIO buffer becomes 6333859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * available, the ACK is sent ahead of other outgoing 6343859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * packets. 6353859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 6363859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell dev->n_rc_qacks++; 6373859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 6383859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbellqueue_ack: 63935ff032e65ab5cc03bbba46cefece7376c7c562fRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 6403859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_flags |= IPATH_S_ACK_PENDING; 6413859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_nak_state = qp->r_nak_state; 6423859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_psn = qp->r_ack_psn; 64335ff032e65ab5cc03bbba46cefece7376c7c562fRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 6443859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 6453859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* Call ipath_do_rc_send() in another thread. */ 6463859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell tasklet_hi_schedule(&qp->s_task); 6473859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 6483859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbelldone: 6493859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell return; 65097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 65197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 65297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 6536700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * reset_psn - reset the QP state to send starting from PSN 6546700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * @qp: the QP 6556700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * @psn: the packet sequence number to restart at 6566700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * 6576700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * This is called from ipath_rc_rcv() to process an incoming RC ACK 6586700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * for the given QP. 6596700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * Called at interrupt level with the QP s_lock held. 6606700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan */ 6616700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivanstatic void reset_psn(struct ipath_qp *qp, u32 psn) 6626700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan{ 6636700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan u32 n = qp->s_last; 6646700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan struct ipath_swqe *wqe = get_swqe_ptr(qp, n); 6656700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan u32 opcode; 6666700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 6676700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_cur = n; 6686700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 6696700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan /* 6706700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * If we are starting the request from the beginning, 6716700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * let the normal send code handle initialization. 6726700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan */ 6736700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (ipath_cmp24(psn, wqe->psn) <= 0) { 6746700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_state = OP(SEND_LAST); 6756700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan goto done; 6766700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan } 6776700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 6786700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan /* Find the work request opcode corresponding to the given PSN. */ 6796700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan opcode = wqe->wr.opcode; 6806700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan for (;;) { 6816700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan int diff; 6826700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 6836700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (++n == qp->s_size) 6846700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan n = 0; 6856700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (n == qp->s_tail) 6866700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan break; 6876700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan wqe = get_swqe_ptr(qp, n); 6886700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan diff = ipath_cmp24(psn, wqe->psn); 6896700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (diff < 0) 6906700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan break; 6916700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_cur = n; 6926700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan /* 6936700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * If we are starting the request from the beginning, 6946700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * let the normal send code handle initialization. 6956700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan */ 6966700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (diff == 0) { 6976700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_state = OP(SEND_LAST); 6986700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan goto done; 6996700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan } 7006700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan opcode = wqe->wr.opcode; 7016700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan } 7026700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 7036700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan /* 7046700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * Set the state to restart in the middle of a request. 7056700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * Don't change the s_sge, s_cur_sge, or s_cur_size. 7066700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * See ipath_do_rc_send(). 7076700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan */ 7086700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan switch (opcode) { 7096700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan case IB_WR_SEND: 7106700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan case IB_WR_SEND_WITH_IMM: 7116700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_state = OP(RDMA_READ_RESPONSE_FIRST); 7126700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan break; 7136700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 7146700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan case IB_WR_RDMA_WRITE: 7156700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan case IB_WR_RDMA_WRITE_WITH_IMM: 7166700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_state = OP(RDMA_READ_RESPONSE_LAST); 7176700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan break; 7186700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 7196700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan case IB_WR_RDMA_READ: 7206700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE); 7216700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan break; 7226700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 7236700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan default: 7246700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan /* 7256700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * This case shouldn't happen since its only 7266700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * one PSN per req. 7276700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan */ 7286700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_state = OP(SEND_LAST); 7296700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan } 7306700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivandone: 7316700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_psn = psn; 7326700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan} 7336700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 7346700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan/** 73597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ipath_restart_rc - back up requester to resend the last un-ACKed request 73697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: the QP to restart 73797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @psn: packet sequence number for the request 73897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @wc: the work completion request 73997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 74012eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * The QP s_lock should be held and interrupts disabled. 74197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 74297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanvoid ipath_restart_rc(struct ipath_qp *qp, u32 psn, struct ib_wc *wc) 74397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 74497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last); 74597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_ibdev *dev; 74697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 74797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_retry == 0) { 74897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->wr_id = wqe->wr.wr_id; 74997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->status = IB_WC_RETRY_EXC_ERR; 75097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->opcode = ib_ipath_wc_opcode[wqe->wr.opcode]; 75197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->vendor_err = 0; 75297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->byte_len = 0; 753062dbb69f32b9ccea701b30f8cc0049482e6211fMichael S. Tsirkin wc->qp = &qp->ibqp; 75497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->src_qp = qp->remote_qpn; 75597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->pkey_index = 0; 75697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->slid = qp->remote_ah_attr.dlid; 75797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->sl = qp->remote_ah_attr.sl; 75897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->dlid_path_bits = 0; 75997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->port_num = 0; 76097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_sqerror_qp(qp, wc); 76197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto bail; 76297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 76397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_retry--; 76497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 76597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 76697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Remove the QP from the timeout queue. 76797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Note: it may already have been removed by ipath_ib_timer(). 76897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 76997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev = to_idev(qp->ibqp.device); 77097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_lock(&dev->pending_lock); 77194b8d9f98d7f535037eb9845b81396f667b4f727Bryan O'Sullivan if (!list_empty(&qp->timerwait)) 77294b8d9f98d7f535037eb9845b81396f667b4f727Bryan O'Sullivan list_del_init(&qp->timerwait); 77397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_unlock(&dev->pending_lock); 77497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 77597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_RDMA_READ) 77697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_rc_resends++; 77797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else 7786f5c407460bba332d6bee52e19f2305539395511Ralph Campbell dev->n_rc_resends += (qp->s_psn - psn) & IPATH_PSN_MASK; 77997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 7806700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan reset_psn(qp, psn); 78197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan tasklet_hi_schedule(&qp->s_task); 78297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 78397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanbail: 78497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan return; 78597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 78697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 7876022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivanstatic inline void update_last_psn(struct ipath_qp *qp, u32 psn) 7886022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan{ 7896022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan if (qp->s_wait_credit) { 7906022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan qp->s_wait_credit = 0; 7916022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan tasklet_hi_schedule(&qp->s_task); 7926022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan } 7936022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan qp->s_last_psn = psn; 7946022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan} 7956022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan 79697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 79797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * do_rc_ack - process an incoming RC ACK 79897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: the QP the ACK came in on 79997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @psn: the packet sequence number of the ACK 80097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @opcode: the opcode of the request that resulted in the ACK 80197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 8026700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * This is called from ipath_rc_rcv_resp() to process an incoming RC ACK 80397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * for the given QP. 80412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * Called at interrupt level with the QP s_lock held and interrupts disabled. 80597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Returns 1 if OK, 0 if current operation should be aborted (NAK). 80697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 80797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanstatic int do_rc_ack(struct ipath_qp *qp, u32 aeth, u32 psn, int opcode) 80897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 80997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_ibdev *dev = to_idev(qp->ibqp.device); 81097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ib_wc wc; 81197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_swqe *wqe; 81297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int ret = 0; 81313b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell u32 ack_psn; 81497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 81597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 81697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Remove the QP from the timeout queue (or RNR timeout queue). 81797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * If ipath_ib_timer() has already removed it, 81897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * it's OK since we hold the QP s_lock and ipath_restart_rc() 81997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * just won't find anything to restart if we ACK everything. 82097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 82197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_lock(&dev->pending_lock); 82294b8d9f98d7f535037eb9845b81396f667b4f727Bryan O'Sullivan if (!list_empty(&qp->timerwait)) 82394b8d9f98d7f535037eb9845b81396f667b4f727Bryan O'Sullivan list_del_init(&qp->timerwait); 82497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_unlock(&dev->pending_lock); 82597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 82697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 82797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Note that NAKs implicitly ACK outstanding SEND and RDMA write 82897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * requests and implicitly NAK RDMA read and atomic requests issued 82997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * before the NAK'ed request. The MSN won't include the NAK'ed 83097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * request but will include an ACK'ed request(s). 83197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 83213b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell ack_psn = psn; 83313b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell if (aeth >> 29) 83413b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell ack_psn--; 83597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe = get_swqe_ptr(qp, qp->s_last); 83697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 83797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 83897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * The MSN might be for a later WQE than the PSN indicates so 83997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * only complete WQEs that the PSN finishes. 84097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 84113b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell while (ipath_cmp24(ack_psn, wqe->lpsn) >= 0) { 84297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 84397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * If this request is a RDMA read or atomic, and the ACK is 84497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * for a later operation, this ACK NAKs the RDMA read or 84597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * atomic. In other words, only a RDMA_READ_LAST or ONLY 84697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * can ACK a RDMA read and likewise for atomic ops. Note 84797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * that the NAK case can only happen if relaxed ordering is 84897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * used and requests are sent after an RDMA read or atomic 84997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * is sent but before the response is received. 85097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 85197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if ((wqe->wr.opcode == IB_WR_RDMA_READ && 85213b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell (opcode != OP(RDMA_READ_RESPONSE_LAST) || 8533859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ipath_cmp24(ack_psn, wqe->lpsn) != 0)) || 85497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 85597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) && 85697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan (opcode != OP(ATOMIC_ACKNOWLEDGE) || 85797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_cmp24(wqe->psn, psn) != 0))) { 85897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 85997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * The last valid PSN seen is the previous 86097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * request's. 86197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 8626022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan update_last_psn(qp, wqe->psn - 1); 86397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Retry this request. */ 86497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_restart_rc(qp, wqe->psn, &wc); 86597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 86697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * No need to process the ACK/NAK since we are 86797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * restarting an earlier request. 86897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 86997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto bail; 87097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 8713859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (qp->s_num_rd_atomic && 8723859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell (wqe->wr.opcode == IB_WR_RDMA_READ || 8733859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 8743859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) { 8753859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_num_rd_atomic--; 8763859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* Restart sending task if fence is complete */ 8773859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if ((qp->s_flags & IPATH_S_FENCE_PENDING) && 8783859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell !qp->s_num_rd_atomic) { 8793859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_flags &= ~IPATH_S_FENCE_PENDING; 8803859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell tasklet_hi_schedule(&qp->s_task); 8813859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } else if (qp->s_flags & IPATH_S_RDMAR_PENDING) { 8823859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_flags &= ~IPATH_S_RDMAR_PENDING; 8833859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell tasklet_hi_schedule(&qp->s_task); 8843859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 8853859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 88697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Post a send completion queue entry if requested. */ 8873859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (!(qp->s_flags & IPATH_S_SIGNAL_REQ_WR) || 88897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan (wqe->wr.send_flags & IB_SEND_SIGNALED)) { 88997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.wr_id = wqe->wr.wr_id; 89097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.status = IB_WC_SUCCESS; 89197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode]; 89297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.vendor_err = 0; 89397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.byte_len = wqe->length; 8940434d271fddaabd65aaa4dbd0145112d6e8aa388Ralph Campbell wc.imm_data = 0; 895062dbb69f32b9ccea701b30f8cc0049482e6211fMichael S. Tsirkin wc.qp = &qp->ibqp; 89697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.src_qp = qp->remote_qpn; 8970434d271fddaabd65aaa4dbd0145112d6e8aa388Ralph Campbell wc.wc_flags = 0; 89897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.pkey_index = 0; 89997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.slid = qp->remote_ah_attr.dlid; 90097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.sl = qp->remote_ah_attr.sl; 90197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.dlid_path_bits = 0; 90297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.port_num = 0; 90397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 0); 90497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 90597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_retry = qp->s_retry_cnt; 90697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 90797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * If we are completing a request which is in the process of 90897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * being resent, we can stop resending it since we know the 90997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * responder has already seen it. 91097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 91197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_last == qp->s_cur) { 91297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (++qp->s_cur >= qp->s_size) 91397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur = 0; 9146f5c407460bba332d6bee52e19f2305539395511Ralph Campbell qp->s_last = qp->s_cur; 9156f5c407460bba332d6bee52e19f2305539395511Ralph Campbell if (qp->s_last == qp->s_tail) 9166f5c407460bba332d6bee52e19f2305539395511Ralph Campbell break; 91797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe = get_swqe_ptr(qp, qp->s_cur); 91897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(SEND_LAST); 91997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_psn = wqe->psn; 9206f5c407460bba332d6bee52e19f2305539395511Ralph Campbell } else { 9216f5c407460bba332d6bee52e19f2305539395511Ralph Campbell if (++qp->s_last >= qp->s_size) 9226f5c407460bba332d6bee52e19f2305539395511Ralph Campbell qp->s_last = 0; 9236f5c407460bba332d6bee52e19f2305539395511Ralph Campbell if (qp->s_last == qp->s_tail) 9246f5c407460bba332d6bee52e19f2305539395511Ralph Campbell break; 9256f5c407460bba332d6bee52e19f2305539395511Ralph Campbell wqe = get_swqe_ptr(qp, qp->s_last); 92697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 92797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 92897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 92997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan switch (aeth >> 29) { 93097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case 0: /* ACK */ 93197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_rc_acks++; 93297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* If this is a partial ACK, reset the retransmit timer. */ 93397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_last != qp->s_tail) { 93497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_lock(&dev->pending_lock); 93597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan list_add_tail(&qp->timerwait, 93697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan &dev->pending[dev->pending_index]); 93797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_unlock(&dev->pending_lock); 9386f5c407460bba332d6bee52e19f2305539395511Ralph Campbell /* 9396f5c407460bba332d6bee52e19f2305539395511Ralph Campbell * If we get a partial ACK for a resent operation, 9406f5c407460bba332d6bee52e19f2305539395511Ralph Campbell * we can stop resending the earlier packets and 9416f5c407460bba332d6bee52e19f2305539395511Ralph Campbell * continue with the next packet the receiver wants. 9426f5c407460bba332d6bee52e19f2305539395511Ralph Campbell */ 9436f5c407460bba332d6bee52e19f2305539395511Ralph Campbell if (ipath_cmp24(qp->s_psn, psn) <= 0) { 9446f5c407460bba332d6bee52e19f2305539395511Ralph Campbell reset_psn(qp, psn + 1); 9456f5c407460bba332d6bee52e19f2305539395511Ralph Campbell tasklet_hi_schedule(&qp->s_task); 9466f5c407460bba332d6bee52e19f2305539395511Ralph Campbell } 9476f5c407460bba332d6bee52e19f2305539395511Ralph Campbell } else if (ipath_cmp24(qp->s_psn, psn) <= 0) { 9486f5c407460bba332d6bee52e19f2305539395511Ralph Campbell qp->s_state = OP(SEND_LAST); 9496f5c407460bba332d6bee52e19f2305539395511Ralph Campbell qp->s_psn = psn + 1; 95097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 95197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_get_credit(qp, aeth); 95297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_rnr_retry = qp->s_rnr_retry_cnt; 95397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_retry = qp->s_retry_cnt; 9546022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan update_last_psn(qp, psn); 95597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ret = 1; 95697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto bail; 95797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 95897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case 1: /* RNR NAK */ 95997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_rnr_naks++; 9606f5c407460bba332d6bee52e19f2305539395511Ralph Campbell if (qp->s_last == qp->s_tail) 9616f5c407460bba332d6bee52e19f2305539395511Ralph Campbell goto bail; 96297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_rnr_retry == 0) { 96397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.status = IB_WC_RNR_RETRY_EXC_ERR; 96497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto class_b; 96597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 96697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_rnr_retry_cnt < 7) 96797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_rnr_retry--; 96897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 9696700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan /* The last valid PSN is the previous PSN. */ 9706022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan update_last_psn(qp, psn - 1); 97197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 9726f5c407460bba332d6bee52e19f2305539395511Ralph Campbell if (wqe->wr.opcode == IB_WR_RDMA_READ) 9736f5c407460bba332d6bee52e19f2305539395511Ralph Campbell dev->n_rc_resends++; 9746f5c407460bba332d6bee52e19f2305539395511Ralph Campbell else 9756f5c407460bba332d6bee52e19f2305539395511Ralph Campbell dev->n_rc_resends += 9766f5c407460bba332d6bee52e19f2305539395511Ralph Campbell (qp->s_psn - psn) & IPATH_PSN_MASK; 97797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 9786700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan reset_psn(qp, psn); 97997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 98097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_rnr_timeout = 98127b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan ib_ipath_rnr_table[(aeth >> IPATH_AETH_CREDIT_SHIFT) & 98227b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan IPATH_AETH_CREDIT_MASK]; 98397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_insert_rnr_queue(qp); 98497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto bail; 98597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 98697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case 3: /* NAK */ 9876f5c407460bba332d6bee52e19f2305539395511Ralph Campbell if (qp->s_last == qp->s_tail) 9886f5c407460bba332d6bee52e19f2305539395511Ralph Campbell goto bail; 9896f5c407460bba332d6bee52e19f2305539395511Ralph Campbell /* The last valid PSN is the previous PSN. */ 9906f5c407460bba332d6bee52e19f2305539395511Ralph Campbell update_last_psn(qp, psn - 1); 99127b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan switch ((aeth >> IPATH_AETH_CREDIT_SHIFT) & 99227b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan IPATH_AETH_CREDIT_MASK) { 99397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case 0: /* PSN sequence error */ 99497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_seq_naks++; 99597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 9966f5c407460bba332d6bee52e19f2305539395511Ralph Campbell * Back up to the responder's expected PSN. 99797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Note that we might get a NAK in the middle of an 99897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * RDMA READ response which terminates the RDMA 99997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * READ. 100097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 100197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_restart_rc(qp, psn, &wc); 100297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 100397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 100497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case 1: /* Invalid Request */ 100597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.status = IB_WC_REM_INV_REQ_ERR; 100697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_other_naks++; 100797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto class_b; 100897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 100997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case 2: /* Remote Access Error */ 101097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.status = IB_WC_REM_ACCESS_ERR; 101197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_other_naks++; 101297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto class_b; 101397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 101497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case 3: /* Remote Operation Error */ 101597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.status = IB_WC_REM_OP_ERR; 101697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_other_naks++; 101797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan class_b: 101897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.wr_id = wqe->wr.wr_id; 101997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode]; 102097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.vendor_err = 0; 102197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.byte_len = 0; 1022062dbb69f32b9ccea701b30f8cc0049482e6211fMichael S. Tsirkin wc.qp = &qp->ibqp; 102397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.src_qp = qp->remote_qpn; 102497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.pkey_index = 0; 102597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.slid = qp->remote_ah_attr.dlid; 102697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.sl = qp->remote_ah_attr.sl; 102797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.dlid_path_bits = 0; 102897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.port_num = 0; 102997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_sqerror_qp(qp, &wc); 103097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 103197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 103297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan default: 103397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Ignore other reserved NAK error codes */ 103497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto reserved; 103597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 103697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_rnr_retry = qp->s_rnr_retry_cnt; 103797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto bail; 103897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 103997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan default: /* 2: reserved */ 104097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan reserved: 104197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Ignore reserved NAK codes. */ 104297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto bail; 104397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 104497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 104597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanbail: 104697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan return ret; 104797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 104897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 104997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 105097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ipath_rc_rcv_resp - process an incoming RC response packet 105197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @dev: the device this packet came in on 105297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @ohdr: the other headers for this packet 105397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @data: the packet data 105497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @tlen: the packet length 105597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: the QP for this packet 105697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @opcode: the opcode for this packet 105797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @psn: the packet sequence number for this packet 105897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @hdrsize: the header length 105997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @pmtu: the path MTU 106097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @header_in_data: true if part of the header data is in the data buffer 106197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 106297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * This is called from ipath_rc_rcv() to process an incoming RC response 106397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * packet for the given QP. 106497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Called at interrupt level. 106597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 106697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanstatic inline void ipath_rc_rcv_resp(struct ipath_ibdev *dev, 106797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_other_headers *ohdr, 106897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan void *data, u32 tlen, 106997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_qp *qp, 107097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 opcode, 107197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 psn, u32 hdrsize, u32 pmtu, 107297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int header_in_data) 107397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 10743859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell struct ipath_swqe *wqe; 107597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan unsigned long flags; 107697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ib_wc wc; 107797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int diff; 107897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 pad; 107997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 aeth; 108097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 108197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_lock_irqsave(&qp->s_lock, flags); 108297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 108397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Ignore invalid responses. */ 108497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (ipath_cmp24(psn, qp->s_next_psn) >= 0) 108597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto ack_done; 108697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 108797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Ignore duplicate responses. */ 108897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan diff = ipath_cmp24(psn, qp->s_last_psn); 108997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(diff <= 0)) { 109097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Update credits for "ghost" ACKs */ 109197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (diff == 0 && opcode == OP(ACKNOWLEDGE)) { 109297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!header_in_data) 109397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan aeth = be32_to_cpu(ohdr->u.aeth); 109497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 109597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan aeth = be32_to_cpu(((__be32 *) data)[0]); 109697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(__be32); 109797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 109897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if ((aeth >> 29) == 0) 109997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_get_credit(qp, aeth); 110097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 110197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto ack_done; 110297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 110397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 11043859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(qp->s_last == qp->s_tail)) 11053859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto ack_done; 11063859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe = get_swqe_ptr(qp, qp->s_last); 11073859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 110897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan switch (opcode) { 110997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(ACKNOWLEDGE): 111097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(ATOMIC_ACKNOWLEDGE): 111197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_FIRST): 111297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!header_in_data) 111397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan aeth = be32_to_cpu(ohdr->u.aeth); 111497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 111597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan aeth = be32_to_cpu(((__be32 *) data)[0]); 111697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(__be32); 111797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 11183859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (opcode == OP(ATOMIC_ACKNOWLEDGE)) { 11193859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u64 val; 11203859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 11213859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (!header_in_data) { 11223859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell __be32 *p = ohdr->u.at.atomic_ack_eth; 11233859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 11243859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell val = ((u64) be32_to_cpu(p[0]) << 32) | 11253859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell be32_to_cpu(p[1]); 11263859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } else 11273859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell val = be64_to_cpu(((__be64 *) data)[0]); 11283859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell *(u64 *) wqe->sg_list[0].vaddr = val; 11293859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 113097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!do_rc_ack(qp, aeth, psn, opcode) || 113197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode != OP(RDMA_READ_RESPONSE_FIRST)) 113297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto ack_done; 113397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hdrsize += 4; 11343859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 1135dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell goto ack_op_err; 113697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 11373859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * If this is a response to a resent RDMA read, we 11383859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * have to be careful to copy the data to the right 11393859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * location. 114097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 11413859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, 11423859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe, psn, pmtu); 11433859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto read_middle; 114497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 114597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_MIDDLE): 114697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* no AETH, no ACK */ 114797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(ipath_cmp24(psn, qp->s_last_psn + 1))) { 114897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_rdma_seq++; 11493859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ipath_restart_rc(qp, qp->s_last_psn + 1, &wc); 115097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto ack_done; 115197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 11523859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 1153dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell goto ack_op_err; 11543859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell read_middle: 11557bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan if (unlikely(tlen != (hdrsize + pmtu + 4))) 1156dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell goto ack_len_err; 11573859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(pmtu >= qp->s_rdma_read_len)) 1158dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell goto ack_len_err; 11593859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 11607bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan /* We got a response so update the timeout. */ 11617bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan spin_lock(&dev->pending_lock); 11627bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan if (qp->s_rnr_timeout == 0 && !list_empty(&qp->timerwait)) 11637bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan list_move_tail(&qp->timerwait, 11647bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan &dev->pending[dev->pending_index]); 11657bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan spin_unlock(&dev->pending_lock); 11667bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan /* 11676700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * Update the RDMA receive state but do the copy w/o 11686700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * holding the locks and blocking interrupts. 11697bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan */ 11703859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_rdma_read_len -= pmtu; 11716022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan update_last_psn(qp, psn); 11727bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan spin_unlock_irqrestore(&qp->s_lock, flags); 11733859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ipath_copy_sge(&qp->s_rdma_read_sge, data, pmtu); 11747bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan goto bail; 117597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 11763859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(RDMA_READ_RESPONSE_ONLY): 11773859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(ipath_cmp24(psn, qp->s_last_psn + 1))) { 11783859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell dev->n_rdma_seq++; 11793859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ipath_restart_rc(qp, qp->s_last_psn + 1, &wc); 11803859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto ack_done; 11813859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 11823859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 1183dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell goto ack_op_err; 1184dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell /* Get the number of bytes the message was padded by. */ 1185dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 1186dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell /* 1187dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell * Check that the data size is >= 0 && <= pmtu. 1188dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell * Remember to account for the AETH header (4) and 1189dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell * ICRC (4). 1190dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell */ 1191dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell if (unlikely(tlen < (hdrsize + pad + 8))) 1192dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell goto ack_len_err; 11933859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 11943859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * If this is a response to a resent RDMA read, we 11953859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * have to be careful to copy the data to the right 11963859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * location. 11973859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 11983859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, 11993859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe, psn, pmtu); 12003859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto read_last; 12013859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 120297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_LAST): 120397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* ACKs READ req. */ 120497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(ipath_cmp24(psn, qp->s_last_psn + 1))) { 120597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_rdma_seq++; 12063859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ipath_restart_rc(qp, qp->s_last_psn + 1, &wc); 120797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto ack_done; 120897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 12093859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 1210dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell goto ack_op_err; 1211dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell /* Get the number of bytes the message was padded by. */ 121297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 121397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 121497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Check that the data size is >= 1 && <= pmtu. 121597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Remember to account for the AETH header (4) and 121697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ICRC (4). 121797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 1218dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell if (unlikely(tlen <= (hdrsize + pad + 8))) 1219dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell goto ack_len_err; 1220dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell read_last: 122197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan tlen -= hdrsize + pad + 8; 1222dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell if (unlikely(tlen != qp->s_rdma_read_len)) 1223dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell goto ack_len_err; 122497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!header_in_data) 122597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan aeth = be32_to_cpu(ohdr->u.aeth); 122697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 122797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan aeth = be32_to_cpu(((__be32 *) data)[0]); 122897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(__be32); 122997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 12303859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ipath_copy_sge(&qp->s_rdma_read_sge, data, tlen); 12313859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell (void) do_rc_ack(qp, aeth, psn, OP(RDMA_READ_RESPONSE_LAST)); 123297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto ack_done; 123397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 123497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 123597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanack_done: 123697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_unlock_irqrestore(&qp->s_lock, flags); 1237dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell goto bail; 1238dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell 1239dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbellack_op_err: 1240dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell wc.status = IB_WC_LOC_QP_OP_ERR; 1241dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell goto ack_err; 1242dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell 1243dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbellack_len_err: 1244dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell wc.status = IB_WC_LOC_LEN_ERR; 1245dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbellack_err: 1246dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell wc.wr_id = wqe->wr.wr_id; 1247dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode]; 1248dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell wc.vendor_err = 0; 1249dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell wc.byte_len = 0; 1250dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell wc.imm_data = 0; 1251dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell wc.qp = &qp->ibqp; 1252dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell wc.src_qp = qp->remote_qpn; 1253dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell wc.wc_flags = 0; 1254dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell wc.pkey_index = 0; 1255dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell wc.slid = qp->remote_ah_attr.dlid; 1256dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell wc.sl = qp->remote_ah_attr.sl; 1257dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell wc.dlid_path_bits = 0; 1258dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell wc.port_num = 0; 1259dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell ipath_sqerror_qp(qp, &wc); 12606ed89b9574776d4178f1ad754d20e4f1e5a4b6c8Ralph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 126197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanbail: 126297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan return; 126397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 126497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 126597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 126697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ipath_rc_rcv_error - process an incoming duplicate or error RC packet 126797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @dev: the device this packet came in on 126897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @ohdr: the other headers for this packet 126997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @data: the packet data 127097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: the QP for this packet 127197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @opcode: the opcode for this packet 127297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @psn: the packet sequence number for this packet 127397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @diff: the difference between the PSN and the expected PSN 127497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @header_in_data: true if part of the header data is in the data buffer 127597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 127697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * This is called from ipath_rc_rcv() to process an unexpected 127797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * incoming RC packet for the given QP. 127897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Called at interrupt level. 127997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Return 1 if no more processing is needed; otherwise return 0 to 12803859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * schedule a response to be sent. 128197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 128297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanstatic inline int ipath_rc_rcv_error(struct ipath_ibdev *dev, 128397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_other_headers *ohdr, 128497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan void *data, 128597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_qp *qp, 128697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 opcode, 128797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 psn, 128897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int diff, 128997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int header_in_data) 129097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 12913859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell struct ipath_ack_entry *e; 12923859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u8 i, prev; 12933859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell int old_req; 129435ff032e65ab5cc03bbba46cefece7376c7c562fRalph Campbell unsigned long flags; 129597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 129697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (diff > 0) { 129797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 129897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Packet sequence error. 129997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * A NAK will ACK earlier sends and RDMA writes. 13003859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Don't queue the NAK if we already sent one. 130197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 13023859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (!qp->r_nak_state) { 130312eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_nak_state = IB_NAK_PSN_ERROR; 130412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan /* Use the expected PSN. */ 130512eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_ack_psn = qp->r_psn; 13063859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto send_ack; 130797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 13083859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto done; 130997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 131097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 131197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 131297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Handle a duplicate request. Don't re-execute SEND, RDMA 131397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * write or atomic op. Don't NAK errors, just silently drop 131497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * the duplicate request. Note that r_sge, r_len, and 131597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * r_rcv_len may be in use so don't modify them. 131697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 131797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * We are supposed to ACK the earliest duplicate PSN but we 131897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * can coalesce an outstanding duplicate ACK. We have to 131997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * send the earliest so that RDMA reads can be restarted at 132097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * the requester's expected PSN. 13213859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * 13223859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * First, find where this duplicate PSN falls within the 13233859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * ACKs previously sent. 132497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 13253859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell psn &= IPATH_PSN_MASK; 13263859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e = NULL; 13273859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell old_req = 1; 132835ff032e65ab5cc03bbba46cefece7376c7c562fRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 13293859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell for (i = qp->r_head_ack_queue; ; i = prev) { 13303859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (i == qp->s_tail_ack_queue) 13313859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell old_req = 0; 13323859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (i) 13333859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell prev = i - 1; 13343859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell else 13353859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell prev = IPATH_MAX_RDMA_ATOMIC; 13363859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (prev == qp->r_head_ack_queue) { 13373859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e = NULL; 13383859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell break; 13393859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 13403859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e = &qp->s_ack_queue[prev]; 13413859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (!e->opcode) { 13423859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e = NULL; 13433859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell break; 13443859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 13453859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (ipath_cmp24(psn, e->psn) >= 0) 13463859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell break; 13473859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 13483859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell switch (opcode) { 13493859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(RDMA_READ_REQUEST): { 13503859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell struct ib_reth *reth; 13513859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u32 offset; 13523859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u32 len; 13533859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 13543859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 13553859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * If we didn't find the RDMA read request in the ack queue, 13563859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * or the send tasklet is already backed up to send an 13573859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * earlier entry, we can ignore this request. 13583859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 13593859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (!e || e->opcode != OP(RDMA_READ_REQUEST) || old_req) 13603859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto unlock_done; 136197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* RETH comes after BTH */ 136297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!header_in_data) 136397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan reth = &ohdr->u.rc.reth; 136497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 136597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan reth = (struct ib_reth *)data; 136697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(*reth); 136797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 136812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan /* 13693859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Address range must be a subset of the original 13703859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * request and start on pmtu boundaries. 13713859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * We reuse the old ack_queue slot since the requester 13723859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * should not back up and request an earlier PSN for the 13733859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * same request. 137412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan */ 13753859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell offset = ((psn - e->psn) & IPATH_PSN_MASK) * 13763859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ib_mtu_enum_to_int(qp->path_mtu); 13773859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = be32_to_cpu(reth->length); 13783859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(offset + len > e->rdma_sge.sge.sge_length)) 13793859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto unlock_done; 13803859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (len != 0) { 138197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 rkey = be32_to_cpu(reth->rkey); 138297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u64 vaddr = be64_to_cpu(reth->vaddr); 138397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int ok; 138497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 13853859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ok = ipath_rkey_ok(qp, &e->rdma_sge, 13863859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len, vaddr, rkey, 138797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan IB_ACCESS_REMOTE_READ); 13883859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(!ok)) 13893859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto unlock_done; 139097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } else { 13913859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sg_list = NULL; 13923859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.num_sge = 0; 13933859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.mr = NULL; 13943859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.vaddr = NULL; 13953859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.length = 0; 13963859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.sge_length = 0; 139797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 13983859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->psn = psn; 13993859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(ACKNOWLEDGE); 14003859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_tail_ack_queue = prev; 14013859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell break; 140212eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan } 140312eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan 140497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(COMPARE_SWAP): 14053859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(FETCH_ADD): { 140697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 14073859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * If we didn't find the atomic request in the ack queue 14083859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * or the send tasklet is already backed up to send an 14093859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * earlier entry, we can ignore this request. 141097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 14113859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (!e || e->opcode != (u8) opcode || old_req) 14123859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto unlock_done; 14133859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(ACKNOWLEDGE); 14143859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_tail_ack_queue = prev; 14153859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell break; 14163859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 14173859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 14183859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell default: 14193859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (old_req) 14203859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto unlock_done; 14213859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 14223859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Resend the most recent ACK if this request is 14233859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * after all the previous RDMA reads and atomics. 14243859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 14253859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (i == qp->r_head_ack_queue) { 142635ff032e65ab5cc03bbba46cefece7376c7c562fRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 14273859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_nak_state = 0; 14283859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_ack_psn = qp->r_psn - 1; 14293859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto send_ack; 14303859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 14313859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 14323859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Resend the RDMA read or atomic op which 14333859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * ACKs this duplicate request. 14343859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 14353859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(ACKNOWLEDGE); 14363859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_tail_ack_queue = i; 143797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 143897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 143912eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_nak_state = 0; 14403859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell tasklet_hi_schedule(&qp->s_task); 144197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 14423859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbellunlock_done: 144335ff032e65ab5cc03bbba46cefece7376c7c562fRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 144497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivandone: 144597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan return 1; 14463859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 14473859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbellsend_ack: 14483859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell return 0; 144997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 145097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 14518d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivanstatic void ipath_rc_error(struct ipath_qp *qp, enum ib_wc_status err) 14528d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivan{ 145335ff032e65ab5cc03bbba46cefece7376c7c562fRalph Campbell unsigned long flags; 145435ff032e65ab5cc03bbba46cefece7376c7c562fRalph Campbell 145535ff032e65ab5cc03bbba46cefece7376c7c562fRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 14568d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivan qp->state = IB_QPS_ERR; 14578d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivan ipath_error_qp(qp, err); 145835ff032e65ab5cc03bbba46cefece7376c7c562fRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 14598d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivan} 14608d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivan 146197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 146297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ipath_rc_rcv - process an incoming RC packet 146397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @dev: the device this packet came in on 146497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @hdr: the header of this packet 146597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @has_grh: true if the header has a GRH 146697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @data: the packet data 146797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @tlen: the packet length 146897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: the QP for this packet 146997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 147097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * This is called from ipath_qp_rcv() to process an incoming RC packet 147197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * for the given QP. 147297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Called at interrupt level. 147397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 147497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanvoid ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 147597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int has_grh, void *data, u32 tlen, struct ipath_qp *qp) 147697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 147797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_other_headers *ohdr; 147897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 opcode; 147997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 hdrsize; 148097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 psn; 148197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 pad; 148297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ib_wc wc; 148397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu); 148497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int diff; 148597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ib_reth *reth; 148697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int header_in_data; 148797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 148810aeb0e6d8823c1cccf9edc8401c848745c128beBryan O'Sullivan /* Validate the SLID. See Ch. 9.6.1.5 */ 148910aeb0e6d8823c1cccf9edc8401c848745c128beBryan O'Sullivan if (unlikely(be16_to_cpu(hdr->lrh[3]) != qp->remote_ah_attr.dlid)) 149010aeb0e6d8823c1cccf9edc8401c848745c128beBryan O'Sullivan goto done; 149110aeb0e6d8823c1cccf9edc8401c848745c128beBryan O'Sullivan 149297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check for GRH */ 149397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!has_grh) { 149497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr = &hdr->u.oth; 149597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hdrsize = 8 + 12; /* LRH + BTH */ 149697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan psn = be32_to_cpu(ohdr->bth[2]); 149797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan header_in_data = 0; 149897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } else { 149997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr = &hdr->u.l.oth; 150097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hdrsize = 8 + 40 + 12; /* LRH + GRH + BTH */ 150197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 150297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * The header with GRH is 60 bytes and the core driver sets 150397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * the eager header buffer size to 56 bytes so the last 4 150497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * bytes of the BTH header (PSN) is in the data buffer. 150597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 150634b2aafea38efdf02cd8107a6e1057e2a297c447Bryan O'Sullivan header_in_data = dev->dd->ipath_rcvhdrentsize == 16; 150797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (header_in_data) { 150897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan psn = be32_to_cpu(((__be32 *) data)[0]); 150997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(__be32); 151097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } else 151197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan psn = be32_to_cpu(ohdr->bth[2]); 151297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 151397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 151497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 151597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Process responses (ACKs) before anything else. Note that the 151697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * packet sequence number will be for something in the send work 151797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * queue rather than the expected receive packet sequence number. 151897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * In other words, this QP is the requester. 151997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 1520ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan opcode = be32_to_cpu(ohdr->bth[0]) >> 24; 152197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) && 152297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode <= OP(ATOMIC_ACKNOWLEDGE)) { 152397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_rc_rcv_resp(dev, ohdr, data, tlen, qp, opcode, psn, 152497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hdrsize, pmtu, header_in_data); 152512eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto done; 152697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 152797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 152897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Compute 24 bits worth of difference. */ 152997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan diff = ipath_cmp24(psn, qp->r_psn); 153097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(diff)) { 153197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (ipath_rc_rcv_error(dev, ohdr, data, qp, opcode, 153297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan psn, diff, header_in_data)) 153397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto done; 153412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto send_ack; 153597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 153697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 153797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check for opcode sequence errors. */ 153897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan switch (qp->r_state) { 153997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_FIRST): 154097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_MIDDLE): 154197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (opcode == OP(SEND_MIDDLE) || 154297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(SEND_LAST) || 154397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(SEND_LAST_WITH_IMMEDIATE)) 154497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 154597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan nack_inv: 15468d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivan ipath_rc_error(qp, IB_WC_REM_INV_REQ_ERR); 154712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_nak_state = IB_NAK_INVALID_REQUEST; 154812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_ack_psn = qp->r_psn; 154912eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto send_ack; 155097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 155197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_FIRST): 155297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_MIDDLE): 155397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (opcode == OP(RDMA_WRITE_MIDDLE) || 155497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(RDMA_WRITE_LAST) || 155597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) 155697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 155797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_inv; 155897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 155997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan default: 156097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (opcode == OP(SEND_MIDDLE) || 156197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(SEND_LAST) || 156297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(SEND_LAST_WITH_IMMEDIATE) || 156397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(RDMA_WRITE_MIDDLE) || 156497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(RDMA_WRITE_LAST) || 156597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) 156697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_inv; 156712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan /* 156812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * Note that it is up to the requester to not send a new 156912eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * RDMA read or atomic operation before receiving an ACK 157012eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * for the previous operation. 157112eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan */ 157297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 157397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 157497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 157597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.imm_data = 0; 157697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.wc_flags = 0; 157797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 157897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* OK, process the packet. */ 157997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan switch (opcode) { 158097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_FIRST): 158197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!ipath_get_rwqe(qp, 0)) { 158297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan rnr_nak: 158397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 158497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * A RNR NAK will ACK earlier sends and RDMA writes. 158597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Don't queue the NAK if a RDMA read or atomic 158697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * is pending though. 158797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 15883859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (qp->r_nak_state) 15893859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto done; 159012eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_nak_state = IB_RNR_NAK | qp->r_min_rnr_timer; 159112eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_ack_psn = qp->r_psn; 159212eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto send_ack; 159397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 159497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_rcv_len = 0; 159597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 159697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_MIDDLE): 159797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_MIDDLE): 159897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan send_middle: 159997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check for invalid length PMTU or posted rwqe len. */ 160097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(tlen != (hdrsize + pmtu + 4))) 160197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_inv; 160297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_rcv_len += pmtu; 160397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(qp->r_rcv_len > qp->r_len)) 160497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_inv; 160597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_copy_sge(&qp->r_sge, data, pmtu); 160697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 160797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 160897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE): 160997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* consume RWQE */ 161097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!ipath_get_rwqe(qp, 1)) 161197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto rnr_nak; 161297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto send_last_imm; 161397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 161497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_ONLY): 161597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_ONLY_WITH_IMMEDIATE): 161697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!ipath_get_rwqe(qp, 0)) 161797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto rnr_nak; 161897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_rcv_len = 0; 161997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (opcode == OP(SEND_ONLY)) 162097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto send_last; 162197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 162297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_LAST_WITH_IMMEDIATE): 162397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan send_last_imm: 162497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (header_in_data) { 162597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.imm_data = *(__be32 *) data; 162697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(__be32); 162797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } else { 162897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Immediate data comes after BTH */ 162997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.imm_data = ohdr->u.imm_data; 163097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 163197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hdrsize += 4; 163297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.wc_flags = IB_WC_WITH_IMM; 163397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 163497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_LAST): 163597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_LAST): 163697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan send_last: 163797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Get the number of bytes the message was padded by. */ 163897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 163997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check for invalid length. */ 164097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* XXX LAST len should be >= 1 */ 164197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(tlen < (hdrsize + pad + 4))) 164297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_inv; 164397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Don't count the CRC. */ 164497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan tlen -= (hdrsize + pad + 4); 164597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.byte_len = tlen + qp->r_rcv_len; 164697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(wc.byte_len > qp->r_len)) 164797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_inv; 164897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_copy_sge(&qp->r_sge, data, tlen); 164912eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_msn++; 16508d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivan if (!qp->r_wrid_valid) 165197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 16528d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivan qp->r_wrid_valid = 0; 165397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.wr_id = qp->r_wr_id; 165497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.status = IB_WC_SUCCESS; 165597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.opcode = IB_WC_RECV; 165697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.vendor_err = 0; 1657062dbb69f32b9ccea701b30f8cc0049482e6211fMichael S. Tsirkin wc.qp = &qp->ibqp; 165897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.src_qp = qp->remote_qpn; 165997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.pkey_index = 0; 166097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.slid = qp->remote_ah_attr.dlid; 166197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.sl = qp->remote_ah_attr.sl; 166297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.dlid_path_bits = 0; 166397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.port_num = 0; 166497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Signal completion event if the solicited bit is set. */ 166597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 166697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan (ohdr->bth[0] & 166797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan __constant_cpu_to_be32(1 << 23)) != 0); 166897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 166997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 167097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_FIRST): 167197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_ONLY): 167297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): 167397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* consume RWQE */ 167497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* RETH comes after BTH */ 167597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!header_in_data) 167697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan reth = &ohdr->u.rc.reth; 167797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 167897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan reth = (struct ib_reth *)data; 167997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(*reth); 168097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 168197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hdrsize += sizeof(*reth); 168297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_len = be32_to_cpu(reth->length); 168397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_rcv_len = 0; 168497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->r_len != 0) { 168597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 rkey = be32_to_cpu(reth->rkey); 168697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u64 vaddr = be64_to_cpu(reth->vaddr); 168797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int ok; 168897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 168997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check rkey & NAK */ 16906a553af286653818bb5831f1b351eefdc8a93b61Bryan O'Sullivan ok = ipath_rkey_ok(qp, &qp->r_sge, 169197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_len, vaddr, rkey, 169297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan IB_ACCESS_REMOTE_WRITE); 169312eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan if (unlikely(!ok)) 169412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto nack_acc; 169597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } else { 169697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_sge.sg_list = NULL; 169797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_sge.sge.mr = NULL; 169897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_sge.sge.vaddr = NULL; 169997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_sge.sge.length = 0; 170097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_sge.sge.sge_length = 0; 170197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 170297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(!(qp->qp_access_flags & 170397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan IB_ACCESS_REMOTE_WRITE))) 170497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_acc; 170597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (opcode == OP(RDMA_WRITE_FIRST)) 170697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto send_middle; 170797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else if (opcode == OP(RDMA_WRITE_ONLY)) 170897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto send_last; 170997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!ipath_get_rwqe(qp, 1)) 171097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto rnr_nak; 171197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto send_last_imm; 171297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 17133859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(RDMA_READ_REQUEST): { 17143859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell struct ipath_ack_entry *e; 17153859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u32 len; 17163859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u8 next; 17173859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 17183859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) 17193859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto nack_acc; 17203859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell next = qp->r_head_ack_queue + 1; 17213859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (next > IPATH_MAX_RDMA_ATOMIC) 17223859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell next = 0; 17233859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(next == qp->s_tail_ack_queue)) 17243859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto nack_inv; 17253859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e = &qp->s_ack_queue[qp->r_head_ack_queue]; 172697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* RETH comes after BTH */ 172797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!header_in_data) 172897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan reth = &ohdr->u.rc.reth; 172997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 173097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan reth = (struct ib_reth *)data; 173197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(*reth); 173297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 17333859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = be32_to_cpu(reth->length); 17343859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (len) { 173597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 rkey = be32_to_cpu(reth->rkey); 173697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u64 vaddr = be64_to_cpu(reth->vaddr); 173797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int ok; 173897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 173997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check rkey & NAK */ 17403859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ok = ipath_rkey_ok(qp, &e->rdma_sge, len, vaddr, 17413859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell rkey, IB_ACCESS_REMOTE_READ); 17423859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(!ok)) 174397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_acc; 174497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 174597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Update the next expected PSN. We add 1 later 174697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * below, so only add the remainder here. 174797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 17483859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (len > pmtu) 17493859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_psn += (len - 1) / pmtu; 175097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } else { 17513859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sg_list = NULL; 17523859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.num_sge = 0; 17533859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.mr = NULL; 17543859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.vaddr = NULL; 17553859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.length = 0; 17563859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.sge_length = 0; 175797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 17583859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->opcode = opcode; 17593859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->psn = psn; 176097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 176197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * We need to increment the MSN here instead of when we 176297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * finish sending the result since a duplicate request would 176397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * increment it more than once. 176497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 176512eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_msn++; 176697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_psn++; 176797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_state = opcode; 176812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_nak_state = 0; 17693859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell barrier(); 17703859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_head_ack_queue = next; 177112eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan 177212eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan /* Call ipath_do_rc_send() in another thread. */ 177312eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan tasklet_hi_schedule(&qp->s_task); 177412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan 177512eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto done; 17763859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 177797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 177897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(COMPARE_SWAP): 177997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(FETCH_ADD): { 178097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ib_atomic_eth *ateth; 17813859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell struct ipath_ack_entry *e; 178297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u64 vaddr; 17833859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell atomic64_t *maddr; 178497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u64 sdata; 178597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 rkey; 17863859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u8 next; 178797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 17883859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(!(qp->qp_access_flags & 17893859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell IB_ACCESS_REMOTE_ATOMIC))) 17903859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto nack_acc; 17913859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell next = qp->r_head_ack_queue + 1; 17923859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (next > IPATH_MAX_RDMA_ATOMIC) 17933859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell next = 0; 17943859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(next == qp->s_tail_ack_queue)) 17953859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto nack_inv; 179697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!header_in_data) 179797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ateth = &ohdr->u.atomic_eth; 17983859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell else 179997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ateth = (struct ib_atomic_eth *)data; 18003859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell vaddr = ((u64) be32_to_cpu(ateth->vaddr[0]) << 32) | 18013859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell be32_to_cpu(ateth->vaddr[1]); 180297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(vaddr & (sizeof(u64) - 1))) 180397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_inv; 180497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan rkey = be32_to_cpu(ateth->rkey); 180597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check rkey & NAK */ 18066a553af286653818bb5831f1b351eefdc8a93b61Bryan O'Sullivan if (unlikely(!ipath_rkey_ok(qp, &qp->r_sge, 180797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan sizeof(u64), vaddr, rkey, 180897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan IB_ACCESS_REMOTE_ATOMIC))) 180997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_acc; 181097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Perform atomic OP and save result. */ 18113859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell maddr = (atomic64_t *) qp->r_sge.sge.vaddr; 181297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan sdata = be64_to_cpu(ateth->swap_data); 18133859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e = &qp->s_ack_queue[qp->r_head_ack_queue]; 18143859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->atomic_data = (opcode == OP(FETCH_ADD)) ? 18153859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell (u64) atomic64_add_return(sdata, maddr) - sdata : 18163859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr, 18173859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell be64_to_cpu(ateth->compare_data), 18183859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell sdata); 18193859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->opcode = opcode; 18203859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->psn = psn & IPATH_PSN_MASK; 182112eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_msn++; 18223859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_psn++; 18233859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_state = opcode; 18243859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_nak_state = 0; 18253859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell barrier(); 18263859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_head_ack_queue = next; 18273859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 18283859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* Call ipath_do_rc_send() in another thread. */ 18293859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell tasklet_hi_schedule(&qp->s_task); 18303859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 18313859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto done; 183297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 183397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 183497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan default: 18353859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* NAK unknown opcodes. */ 18363859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto nack_inv; 183797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 183897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_psn++; 183997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_state = opcode; 18403859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_ack_psn = psn; 184112eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_nak_state = 0; 184297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Send an ACK if requested or required. */ 18433859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (psn & (1 << 31)) 184412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto send_ack; 184512eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto done; 184697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 184712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivannack_acc: 18483859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ipath_rc_error(qp, IB_WC_REM_ACCESS_ERR); 18493859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; 18503859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_ack_psn = qp->r_psn; 18513859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 185212eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivansend_ack: 18533859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell send_rc_ack(qp); 185497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 185512eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivandone: 185697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan return; 185797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 1858