ipath_rc.c revision 3859e39d75b72f35f7d38c618fbbacb39a440c22
197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/* 2759d57686dab8169ca68bbf938ce8e965d1e107aBryan O'Sullivan * Copyright (c) 2006 QLogic, Inc. All rights reserved. 397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. 497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * This software is available to you under a choice of one of two 697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * licenses. You may choose to be licensed under the terms of the GNU 797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * General Public License (GPL) Version 2, available from the file 897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * COPYING in the main directory of this source tree, or the 997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * OpenIB.org BSD license below: 1097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 1197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Redistribution and use in source and binary forms, with or 1297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * without modification, are permitted provided that the following 1397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * conditions are met: 1497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 1597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * - Redistributions of source code must retain the above 1697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * copyright notice, this list of conditions and the following 1797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * disclaimer. 1897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 1997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * - Redistributions in binary form must reproduce the above 2097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * copyright notice, this list of conditions and the following 2197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * disclaimer in the documentation and/or other materials 2297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * provided with the distribution. 2397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 2497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 2597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 2697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 2797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 2897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 2997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 3097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 3197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * SOFTWARE. 3297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 3397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 3497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan#include "ipath_verbs.h" 3534b2aafea38efdf02cd8107a6e1057e2a297c447Bryan O'Sullivan#include "ipath_kernel.h" 3697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 3797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/* cut down ridiculously long IB macro names */ 3897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan#define OP(x) IB_OPCODE_RC_##x 3997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 403859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbellstatic u32 restart_sge(struct ipath_sge_state *ss, struct ipath_swqe *wqe, 413859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u32 psn, u32 pmtu) 423859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell{ 433859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u32 len; 443859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 453859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = ((psn - wqe->psn) & IPATH_PSN_MASK) * pmtu; 463859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ss->sge = wqe->sg_list[0]; 473859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ss->sg_list = wqe->sg_list + 1; 483859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ss->num_sge = wqe->wr.num_sge; 493859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ipath_skip_sge(ss, len); 503859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell return wqe->length - len; 513859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell} 523859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 5397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 5497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ipath_init_restart- initialize the qp->s_sge after a restart 5597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: the QP who's SGE we're restarting 5697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @wqe: the work queue to initialize the QP's SGE from 5797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 5812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * The QP s_lock should be held and interrupts disabled. 5997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 6097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanstatic void ipath_init_restart(struct ipath_qp *qp, struct ipath_swqe *wqe) 6197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 6297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_ibdev *dev; 6397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 643859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, 653859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ib_mtu_enum_to_int(qp->path_mtu)); 6697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev = to_idev(qp->ibqp.device); 6797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_lock(&dev->pending_lock); 6894b8d9f98d7f535037eb9845b81396f667b4f727Bryan O'Sullivan if (list_empty(&qp->timerwait)) 6997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan list_add_tail(&qp->timerwait, 7097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan &dev->pending[dev->pending_index]); 7197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_unlock(&dev->pending_lock); 7297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 7397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 7497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 7597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ipath_make_rc_ack - construct a response packet (ACK, NAK, or RDMA read) 7697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: a pointer to the QP 7797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @ohdr: a pointer to the IB header being constructed 7897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @pmtu: the path MTU 7997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 803859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Return 1 if constructed; otherwise, return 0. 813859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Note that we are in the responder's side of the QP context. 8297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Note the QP s_lock must be held. 8397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 843859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbellstatic int ipath_make_rc_ack(struct ipath_qp *qp, 853859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell struct ipath_other_headers *ohdr, 863859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u32 pmtu, u32 *bth0p, u32 *bth2p) 8797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 883859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell struct ipath_ack_entry *e; 8997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 hwords; 9097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 len; 9197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 bth0; 923859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u32 bth2; 9397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 9497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* header size in 32-bit words LRH+BTH = (8+12)/4. */ 9597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords = 5; 9697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 9797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan switch (qp->s_ack_state) { 983859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(RDMA_READ_RESPONSE_LAST): 993859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(RDMA_READ_RESPONSE_ONLY): 1003859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(ATOMIC_ACKNOWLEDGE): 1013859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(ACKNOWLEDGE); 1023859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* FALLTHROUGH */ 1033859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(ACKNOWLEDGE): 1043859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* Check for no next entry in the queue. */ 1053859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (qp->r_head_ack_queue == qp->s_tail_ack_queue) { 1063859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (qp->s_flags & IPATH_S_ACK_PENDING) 1073859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto normal; 1083859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 1093859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 1103859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 1113859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e = &qp->s_ack_queue[qp->s_tail_ack_queue]; 1123859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (e->opcode == OP(RDMA_READ_REQUEST)) { 1133859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* Copy SGE state in case we need to resend */ 1143859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_rdma_sge = e->rdma_sge; 1153859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_cur_sge = &qp->s_ack_rdma_sge; 1163859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = e->rdma_sge.sge.sge_length; 1173859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (len > pmtu) { 1183859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = pmtu; 1193859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST); 1203859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } else { 1213859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY); 1223859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (++qp->s_tail_ack_queue > 1233859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell IPATH_MAX_RDMA_ATOMIC) 1243859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_tail_ack_queue = 0; 1253859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 1263859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.aeth = ipath_compute_aeth(qp); 1273859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell hwords++; 1283859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_rdma_psn = e->psn; 1293859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell bth2 = qp->s_ack_rdma_psn++ & IPATH_PSN_MASK; 1303859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } else { 1313859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* COMPARE_SWAP or FETCH_ADD */ 1323859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_cur_sge = NULL; 1333859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = 0; 1343859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); 1353859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.at.aeth = ipath_compute_aeth(qp); 1363859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.at.atomic_ack_eth[0] = 1373859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell cpu_to_be32(e->atomic_data >> 32); 1383859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.at.atomic_ack_eth[1] = 1393859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell cpu_to_be32(e->atomic_data); 1403859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell hwords += sizeof(ohdr->u.at) / sizeof(u32); 1413859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell bth2 = e->psn; 1423859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (++qp->s_tail_ack_queue > IPATH_MAX_RDMA_ATOMIC) 1433859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_tail_ack_queue = 0; 1443859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 14597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth0 = qp->s_ack_state << 24; 14697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 14797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 14897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_FIRST): 14997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE); 15097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 15197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_MIDDLE): 1523859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = qp->s_ack_rdma_sge.sge.sge_length; 15397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (len > pmtu) 15497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = pmtu; 15597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 15697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.aeth = ipath_compute_aeth(qp); 15797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords++; 15897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST); 1593859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (++qp->s_tail_ack_queue > IPATH_MAX_RDMA_ATOMIC) 1603859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_tail_ack_queue = 0; 16197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 16297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth0 = qp->s_ack_state << 24; 1633859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell bth2 = qp->s_ack_rdma_psn++ & IPATH_PSN_MASK; 16497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 16597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 16697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan default: 1673859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell normal: 16812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan /* 1693859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Send a regular ACK. 1703859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Set the s_ack_state so we wait until after sending 1713859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * the ACK before setting s_ack_state to ACKNOWLEDGE 1723859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * (see above). 17312eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan */ 1743859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); 1753859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_flags &= ~IPATH_S_ACK_PENDING; 1763859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_cur_sge = NULL; 17712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan if (qp->s_nak_state) 1783859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.aeth = 1793859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell cpu_to_be32((qp->r_msn & IPATH_MSN_MASK) | 1803859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell (qp->s_nak_state << 1813859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell IPATH_AETH_CREDIT_SHIFT)); 18212eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan else 18312eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan ohdr->u.aeth = ipath_compute_aeth(qp); 18497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords++; 1853859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = 0; 1863859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell bth0 = OP(ACKNOWLEDGE) << 24; 1873859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell bth2 = qp->s_ack_psn & IPATH_PSN_MASK; 18897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 18997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_hdrwords = hwords; 19097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur_size = len; 1913859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell *bth0p = bth0; 1923859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell *bth2p = bth2; 1933859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell return 1; 19497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 19512eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivanbail: 1963859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell return 0; 19797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 19897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 19997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 20097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ipath_make_rc_req - construct a request packet (SEND, RDMA r/w, ATOMIC) 20197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: a pointer to the QP 20297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @ohdr: a pointer to the IB header being constructed 20397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @pmtu: the path MTU 20497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @bth0p: pointer to the BTH opcode word 20597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @bth2p: pointer to the BTH PSN word 20697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 20797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Return 1 if constructed; otherwise, return 0. 20812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * Note the QP s_lock must be held and interrupts disabled. 20997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 210ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivanint ipath_make_rc_req(struct ipath_qp *qp, 211ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan struct ipath_other_headers *ohdr, 212ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan u32 pmtu, u32 *bth0p, u32 *bth2p) 21397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 21497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_ibdev *dev = to_idev(qp->ibqp.device); 21597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_sge_state *ss; 21697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_swqe *wqe; 21797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 hwords; 21897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 len; 21997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 bth0; 22097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 bth2; 22197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan char newreq; 22297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 2233859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* Sending responses has higher priority over sending requests. */ 2243859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if ((qp->r_head_ack_queue != qp->s_tail_ack_queue || 2253859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell (qp->s_flags & IPATH_S_ACK_PENDING) || 2263859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state != IB_OPCODE_RC_ACKNOWLEDGE) && 2273859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ipath_make_rc_ack(qp, ohdr, pmtu, bth0p, bth2p)) 2283859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto done; 2293859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 23097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK) || 23197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_rnr_timeout) 2323859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 23397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 2346022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan /* Limit the number of packets sent without an ACK. */ 2356022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan if (ipath_cmp24(qp->s_psn, qp->s_last_psn + IPATH_PSN_CREDIT) > 0) { 2366022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan qp->s_wait_credit = 1; 2376022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan dev->n_rc_stalls++; 2386022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan spin_lock(&dev->pending_lock); 2396022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan if (list_empty(&qp->timerwait)) 2406022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan list_add_tail(&qp->timerwait, 2416022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan &dev->pending[dev->pending_index]); 2426022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan spin_unlock(&dev->pending_lock); 2433859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 2446022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan } 2456022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan 24697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* header size in 32-bit words LRH+BTH = (8+12)/4. */ 24797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords = 5; 24897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth0 = 0; 24997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 25097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Send a request. */ 25197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe = get_swqe_ptr(qp, qp->s_cur); 25297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan switch (qp->s_state) { 25397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan default: 25497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 25597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Resend an old request or start a new one. 25697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 25797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * We keep track of the current SWQE so that 25897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * we don't reset the "furthest progress" state 25997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * if we need to back up. 26097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 26197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan newreq = 0; 26297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_cur == qp->s_tail) { 26397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check if send work queue is empty. */ 26497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_tail == qp->s_head) 2653859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 2663859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 2673859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * If a fence is requested, wait for previous 2683859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * RDMA read and atomic operations to finish. 2693859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 2703859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if ((wqe->wr.send_flags & IB_SEND_FENCE) && 2713859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_num_rd_atomic) { 2723859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_flags |= IPATH_S_FENCE_PENDING; 2733859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 2743859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 2756022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan wqe->psn = qp->s_next_psn; 27697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan newreq = 1; 27797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 27897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 27997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Note that we have to be careful not to modify the 28097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * original work request since we may need to resend 28197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * it. 28297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 28313b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell len = wqe->length; 28497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ss = &qp->s_sge; 28597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth2 = 0; 28697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan switch (wqe->wr.opcode) { 28797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case IB_WR_SEND: 28897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case IB_WR_SEND_WITH_IMM: 28997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* If no credit, return. */ 29097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_lsn != (u32) -1 && 29197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) 2923859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 29397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe->lpsn = wqe->psn; 29497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (len > pmtu) { 29597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe->lpsn += (len - 1) / pmtu; 29697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(SEND_FIRST); 29797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = pmtu; 29897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 29997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 30097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_SEND) 30197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(SEND_ONLY); 30297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 30397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE); 30497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Immediate data comes after the BTH */ 30597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.imm_data = wqe->wr.imm_data; 30697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords += 1; 30797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 30897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.send_flags & IB_SEND_SOLICITED) 30997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth0 |= 1 << 23; 31097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth2 = 1 << 31; /* Request ACK. */ 31197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (++qp->s_cur == qp->s_size) 31297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur = 0; 31397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 31497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 31597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case IB_WR_RDMA_WRITE: 3166700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (newreq && qp->s_lsn != (u32) -1) 31797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_lsn++; 31897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 31997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case IB_WR_RDMA_WRITE_WITH_IMM: 32097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* If no credit, return. */ 32197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_lsn != (u32) -1 && 32297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) 3233859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 32497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.rc.reth.vaddr = 32597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan cpu_to_be64(wqe->wr.wr.rdma.remote_addr); 32697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.rc.reth.rkey = 32797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan cpu_to_be32(wqe->wr.wr.rdma.rkey); 32897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.rc.reth.length = cpu_to_be32(len); 3293859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell hwords += sizeof(struct ib_reth) / sizeof(u32); 33097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe->lpsn = wqe->psn; 33197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (len > pmtu) { 33297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe->lpsn += (len - 1) / pmtu; 33397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(RDMA_WRITE_FIRST); 33497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = pmtu; 33597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 33697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 33797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_RDMA_WRITE) 33897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(RDMA_WRITE_ONLY); 33997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 34097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = 34197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE); 3426700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan /* Immediate data comes after RETH */ 34397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.rc.imm_data = wqe->wr.imm_data; 34497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords += 1; 34597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.send_flags & IB_SEND_SOLICITED) 34697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth0 |= 1 << 23; 34797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 34897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth2 = 1 << 31; /* Request ACK. */ 34997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (++qp->s_cur == qp->s_size) 35097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur = 0; 35197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 35297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 35397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case IB_WR_RDMA_READ: 3543859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 3553859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Don't allow more operations to be started 3563859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * than the QP limits allow. 3573859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 35897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (newreq) { 3593859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (qp->s_num_rd_atomic >= 3603859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_max_rd_atomic) { 3613859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_flags |= IPATH_S_RDMAR_PENDING; 3623859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 3633859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 3643859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_num_rd_atomic++; 3656700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (qp->s_lsn != (u32) -1) 3666700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_lsn++; 36797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 36897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Adjust s_next_psn to count the 36997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * expected number of responses. 37097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 37197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (len > pmtu) 37297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_next_psn += (len - 1) / pmtu; 37397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe->lpsn = qp->s_next_psn++; 37497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 3753859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.rc.reth.vaddr = 3763859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell cpu_to_be64(wqe->wr.wr.rdma.remote_addr); 3773859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.rc.reth.rkey = 3783859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell cpu_to_be32(wqe->wr.wr.rdma.rkey); 3793859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.rc.reth.length = cpu_to_be32(len); 3803859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_state = OP(RDMA_READ_REQUEST); 3813859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32); 38297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ss = NULL; 38397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = 0; 38497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (++qp->s_cur == qp->s_size) 38597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur = 0; 38697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 38797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 38897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case IB_WR_ATOMIC_CMP_AND_SWP: 38997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case IB_WR_ATOMIC_FETCH_AND_ADD: 3903859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 3913859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Don't allow more operations to be started 3923859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * than the QP limits allow. 3933859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 39497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (newreq) { 3953859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (qp->s_num_rd_atomic >= 3963859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_max_rd_atomic) { 3973859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_flags |= IPATH_S_RDMAR_PENDING; 3983859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 3993859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 4003859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_num_rd_atomic++; 4016700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (qp->s_lsn != (u32) -1) 4026700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_lsn++; 40397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe->lpsn = wqe->psn; 40497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 4053859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) { 4063859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_state = OP(COMPARE_SWAP); 4073859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.atomic_eth.swap_data = cpu_to_be64( 4083859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.wr.atomic.swap); 4093859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.atomic_eth.compare_data = cpu_to_be64( 4103859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.wr.atomic.compare_add); 4113859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } else { 4123859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_state = OP(FETCH_ADD); 4133859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.atomic_eth.swap_data = cpu_to_be64( 4143859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.wr.atomic.compare_add); 4153859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.atomic_eth.compare_data = 0; 4163859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 4173859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.atomic_eth.vaddr[0] = cpu_to_be32( 4183859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.wr.atomic.remote_addr >> 32); 4193859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.atomic_eth.vaddr[1] = cpu_to_be32( 4203859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.wr.atomic.remote_addr); 4213859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.atomic_eth.rkey = cpu_to_be32( 4223859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.wr.atomic.rkey); 4233859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell hwords += sizeof(struct ib_atomic_eth) / sizeof(u32); 42497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ss = NULL; 42597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = 0; 4263859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (++qp->s_cur == qp->s_size) 4273859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_cur = 0; 42897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 42997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 43097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan default: 4313859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 43297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 43313b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell qp->s_sge.sge = wqe->sg_list[0]; 43413b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell qp->s_sge.sg_list = wqe->sg_list + 1; 43513b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell qp->s_sge.num_sge = wqe->wr.num_sge; 43613b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell qp->s_len = wqe->length; 43797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (newreq) { 43897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_tail++; 43997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_tail >= qp->s_size) 44097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_tail = 0; 44197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 44213b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell bth2 |= qp->s_psn & IPATH_PSN_MASK; 44313b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell if (wqe->wr.opcode == IB_WR_RDMA_READ) 44413b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell qp->s_psn = wqe->lpsn + 1; 44513b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell else { 44613b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell qp->s_psn++; 44713b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell if ((int)(qp->s_psn - qp->s_next_psn) > 0) 44813b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell qp->s_next_psn = qp->s_psn; 44913b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell } 45012eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan /* 45112eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * Put the QP on the pending list so lost ACKs will cause 45212eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * a retry. More than one request can be pending so the 45312eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * QP may already be on the dev->pending list. 45412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan */ 45597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_lock(&dev->pending_lock); 45694b8d9f98d7f535037eb9845b81396f667b4f727Bryan O'Sullivan if (list_empty(&qp->timerwait)) 45797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan list_add_tail(&qp->timerwait, 45897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan &dev->pending[dev->pending_index]); 45997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_unlock(&dev->pending_lock); 46097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 46197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 46297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_FIRST): 46397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 46412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * This case can only happen if a send is restarted. 46512eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * See ipath_restart_rc(). 46697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 46797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_init_restart(qp, wqe); 46897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 46997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_FIRST): 47097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(SEND_MIDDLE); 47197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 47297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_MIDDLE): 47327b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan bth2 = qp->s_psn++ & IPATH_PSN_MASK; 47497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if ((int)(qp->s_psn - qp->s_next_psn) > 0) 47597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_next_psn = qp->s_psn; 47697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ss = &qp->s_sge; 47797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = qp->s_len; 47897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (len > pmtu) { 47997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = pmtu; 48097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 48197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 48297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_SEND) 48397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(SEND_LAST); 48497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 48597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); 48697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Immediate data comes after the BTH */ 48797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.imm_data = wqe->wr.imm_data; 48897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords += 1; 48997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 49097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.send_flags & IB_SEND_SOLICITED) 49197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth0 |= 1 << 23; 49297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth2 |= 1 << 31; /* Request ACK. */ 49397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur++; 49497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_cur >= qp->s_size) 49597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur = 0; 49697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 49797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 49897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_LAST): 49997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 50097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * This case can only happen if a RDMA write is restarted. 50197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * See ipath_restart_rc(). 50297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 50397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_init_restart(qp, wqe); 50497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 50597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_FIRST): 50697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(RDMA_WRITE_MIDDLE); 50797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 50897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_MIDDLE): 50927b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan bth2 = qp->s_psn++ & IPATH_PSN_MASK; 51097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if ((int)(qp->s_psn - qp->s_next_psn) > 0) 51197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_next_psn = qp->s_psn; 51297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ss = &qp->s_sge; 51397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = qp->s_len; 51497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (len > pmtu) { 51597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = pmtu; 51697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 51797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 51897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_RDMA_WRITE) 51997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(RDMA_WRITE_LAST); 52097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 52197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); 52297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Immediate data comes after the BTH */ 52397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.imm_data = wqe->wr.imm_data; 52497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords += 1; 52597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.send_flags & IB_SEND_SOLICITED) 52697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth0 |= 1 << 23; 52797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 52897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth2 |= 1 << 31; /* Request ACK. */ 52997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur++; 53097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_cur >= qp->s_size) 53197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur = 0; 53297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 53397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 53497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_MIDDLE): 53597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 53697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * This case can only happen if a RDMA read is restarted. 53797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * See ipath_restart_rc(). 53897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 53997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_init_restart(qp, wqe); 54027b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan len = ((qp->s_psn - wqe->psn) & IPATH_PSN_MASK) * pmtu; 54197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.rc.reth.vaddr = 54297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan cpu_to_be64(wqe->wr.wr.rdma.remote_addr + len); 54397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.rc.reth.rkey = 54497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan cpu_to_be32(wqe->wr.wr.rdma.rkey); 54597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.rc.reth.length = cpu_to_be32(qp->s_len); 54697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(RDMA_READ_REQUEST); 5473859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32); 54827b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan bth2 = qp->s_psn++ & IPATH_PSN_MASK; 54997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if ((int)(qp->s_psn - qp->s_next_psn) > 0) 55097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_next_psn = qp->s_psn; 55197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ss = NULL; 55297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = 0; 55397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur++; 55497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_cur == qp->s_size) 55597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur = 0; 55697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 55797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 5586022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan if (ipath_cmp24(qp->s_psn, qp->s_last_psn + IPATH_PSN_CREDIT - 1) >= 0) 5596022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan bth2 |= 1 << 31; /* Request ACK. */ 56097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_len -= len; 56197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_hdrwords = hwords; 56297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur_sge = ss; 56397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur_size = len; 56497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan *bth0p = bth0 | (qp->s_state << 24); 56597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan *bth2p = bth2; 5663859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbelldone: 56797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan return 1; 56897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 5693859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbellbail: 57097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan return 0; 57197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 57297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 57397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 574ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * send_rc_ack - Construct an ACK packet and send it 575ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * @qp: a pointer to the QP 57697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 577ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * This is called from ipath_rc_rcv() and only uses the receive 578ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * side QP state. 5793859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Note that RDMA reads and atomics are handled in the 5803859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * send side QP state and tasklet. 58197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 58297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanstatic void send_rc_ack(struct ipath_qp *qp) 58397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 58497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_ibdev *dev = to_idev(qp->ibqp.device); 58597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u16 lrh0; 58697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 bth0; 587ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan u32 hwords; 588ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan struct ipath_ib_header hdr; 58997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_other_headers *ohdr; 59097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 5913859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* Don't send ACK or NAK if a RDMA read or atomic is pending. */ 5923859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (qp->r_head_ack_queue != qp->s_tail_ack_queue) 5933859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto queue_ack; 5943859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 59597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Construct the header. */ 596ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan ohdr = &hdr.u.oth; 59727b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan lrh0 = IPATH_LRH_BTH; 59897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* header size in 32-bit words LRH+BTH+AETH = (8+12+4)/4. */ 599ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hwords = 6; 60097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) { 601ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hwords += ipath_make_grh(dev, &hdr.u.l.grh, 602ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan &qp->remote_ah_attr.grh, 603ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hwords, 0); 604ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan ohdr = &hdr.u.l.oth; 60527b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan lrh0 = IPATH_LRH_GRH; 60697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 60712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan /* read pkey_index w/o lock (its atomic) */ 6083859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell bth0 = ipath_get_pkey(dev->dd, qp->s_pkey_index) | 6093859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell OP(ACKNOWLEDGE) << 24; 61012eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan if (qp->r_nak_state) 61127b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan ohdr->u.aeth = cpu_to_be32((qp->r_msn & IPATH_MSN_MASK) | 61212eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan (qp->r_nak_state << 61327b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan IPATH_AETH_CREDIT_SHIFT)); 61412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan else 61512eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan ohdr->u.aeth = ipath_compute_aeth(qp); 61697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan lrh0 |= qp->remote_ah_attr.sl << 4; 617ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hdr.lrh[0] = cpu_to_be16(lrh0); 618ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid); 619ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hdr.lrh[2] = cpu_to_be16(hwords + SIZE_OF_CRC); 62034b2aafea38efdf02cd8107a6e1057e2a297c447Bryan O'Sullivan hdr.lrh[3] = cpu_to_be16(dev->dd->ipath_lid); 62197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->bth[0] = cpu_to_be32(bth0); 62297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->bth[1] = cpu_to_be32(qp->remote_qpn); 62327b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan ohdr->bth[2] = cpu_to_be32(qp->r_ack_psn & IPATH_PSN_MASK); 62497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 62597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 62697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * If we can send the ACK, clear the ACK state. 62797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 628ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan if (ipath_verbs_send(dev->dd, hwords, (u32 *) &hdr, 0, NULL) == 0) { 62997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_unicast_xmit++; 6303859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto done; 63197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 6323859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 6333859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 6343859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * We are out of PIO buffers at the moment. 6353859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Pass responsibility for sending the ACK to the 6363859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * send tasklet so that when a PIO buffer becomes 6373859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * available, the ACK is sent ahead of other outgoing 6383859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * packets. 6393859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 6403859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell dev->n_rc_qacks++; 6413859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 6423859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbellqueue_ack: 6433859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell spin_lock_irq(&qp->s_lock); 6443859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_flags |= IPATH_S_ACK_PENDING; 6453859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_nak_state = qp->r_nak_state; 6463859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_psn = qp->r_ack_psn; 6473859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell spin_unlock_irq(&qp->s_lock); 6483859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 6493859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* Call ipath_do_rc_send() in another thread. */ 6503859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell tasklet_hi_schedule(&qp->s_task); 6513859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 6523859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbelldone: 6533859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell return; 65497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 65597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 65697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 6576700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * reset_psn - reset the QP state to send starting from PSN 6586700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * @qp: the QP 6596700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * @psn: the packet sequence number to restart at 6606700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * 6616700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * This is called from ipath_rc_rcv() to process an incoming RC ACK 6626700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * for the given QP. 6636700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * Called at interrupt level with the QP s_lock held. 6646700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan */ 6656700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivanstatic void reset_psn(struct ipath_qp *qp, u32 psn) 6666700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan{ 6676700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan u32 n = qp->s_last; 6686700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan struct ipath_swqe *wqe = get_swqe_ptr(qp, n); 6696700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan u32 opcode; 6706700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 6716700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_cur = n; 6726700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 6736700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan /* 6746700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * If we are starting the request from the beginning, 6756700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * let the normal send code handle initialization. 6766700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan */ 6776700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (ipath_cmp24(psn, wqe->psn) <= 0) { 6786700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_state = OP(SEND_LAST); 6796700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan goto done; 6806700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan } 6816700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 6826700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan /* Find the work request opcode corresponding to the given PSN. */ 6836700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan opcode = wqe->wr.opcode; 6846700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan for (;;) { 6856700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan int diff; 6866700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 6876700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (++n == qp->s_size) 6886700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan n = 0; 6896700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (n == qp->s_tail) 6906700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan break; 6916700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan wqe = get_swqe_ptr(qp, n); 6926700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan diff = ipath_cmp24(psn, wqe->psn); 6936700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (diff < 0) 6946700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan break; 6956700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_cur = n; 6966700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan /* 6976700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * If we are starting the request from the beginning, 6986700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * let the normal send code handle initialization. 6996700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan */ 7006700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (diff == 0) { 7016700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_state = OP(SEND_LAST); 7026700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan goto done; 7036700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan } 7046700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan opcode = wqe->wr.opcode; 7056700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan } 7066700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 7076700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan /* 7086700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * Set the state to restart in the middle of a request. 7096700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * Don't change the s_sge, s_cur_sge, or s_cur_size. 7106700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * See ipath_do_rc_send(). 7116700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan */ 7126700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan switch (opcode) { 7136700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan case IB_WR_SEND: 7146700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan case IB_WR_SEND_WITH_IMM: 7156700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_state = OP(RDMA_READ_RESPONSE_FIRST); 7166700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan break; 7176700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 7186700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan case IB_WR_RDMA_WRITE: 7196700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan case IB_WR_RDMA_WRITE_WITH_IMM: 7206700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_state = OP(RDMA_READ_RESPONSE_LAST); 7216700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan break; 7226700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 7236700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan case IB_WR_RDMA_READ: 7246700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE); 7256700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan break; 7266700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 7276700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan default: 7286700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan /* 7296700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * This case shouldn't happen since its only 7306700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * one PSN per req. 7316700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan */ 7326700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_state = OP(SEND_LAST); 7336700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan } 7346700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivandone: 7356700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_psn = psn; 7366700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan} 7376700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 7386700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan/** 73997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ipath_restart_rc - back up requester to resend the last un-ACKed request 74097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: the QP to restart 74197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @psn: packet sequence number for the request 74297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @wc: the work completion request 74397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 74412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * The QP s_lock should be held and interrupts disabled. 74597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 74697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanvoid ipath_restart_rc(struct ipath_qp *qp, u32 psn, struct ib_wc *wc) 74797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 74897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last); 74997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_ibdev *dev; 75097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 75197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_retry == 0) { 75297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->wr_id = wqe->wr.wr_id; 75397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->status = IB_WC_RETRY_EXC_ERR; 75497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->opcode = ib_ipath_wc_opcode[wqe->wr.opcode]; 75597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->vendor_err = 0; 75697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->byte_len = 0; 757062dbb69f32b9ccea701b30f8cc0049482e6211fMichael S. Tsirkin wc->qp = &qp->ibqp; 75897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->src_qp = qp->remote_qpn; 75997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->pkey_index = 0; 76097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->slid = qp->remote_ah_attr.dlid; 76197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->sl = qp->remote_ah_attr.sl; 76297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->dlid_path_bits = 0; 76397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc->port_num = 0; 76497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_sqerror_qp(qp, wc); 76597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto bail; 76697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 76797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_retry--; 76897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 76997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 77097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Remove the QP from the timeout queue. 77197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Note: it may already have been removed by ipath_ib_timer(). 77297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 77397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev = to_idev(qp->ibqp.device); 77497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_lock(&dev->pending_lock); 77594b8d9f98d7f535037eb9845b81396f667b4f727Bryan O'Sullivan if (!list_empty(&qp->timerwait)) 77694b8d9f98d7f535037eb9845b81396f667b4f727Bryan O'Sullivan list_del_init(&qp->timerwait); 77797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_unlock(&dev->pending_lock); 77897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 77997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_RDMA_READ) 78097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_rc_resends++; 78197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else 78297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_rc_resends += (int)qp->s_psn - (int)psn; 78397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 7846700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan reset_psn(qp, psn); 78597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan tasklet_hi_schedule(&qp->s_task); 78697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 78797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanbail: 78897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan return; 78997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 79097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 7916022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivanstatic inline void update_last_psn(struct ipath_qp *qp, u32 psn) 7926022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan{ 7936022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan if (qp->s_wait_credit) { 7946022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan qp->s_wait_credit = 0; 7956022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan tasklet_hi_schedule(&qp->s_task); 7966022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan } 7976022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan qp->s_last_psn = psn; 7986022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan} 7996022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan 80097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 80197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * do_rc_ack - process an incoming RC ACK 80297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: the QP the ACK came in on 80397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @psn: the packet sequence number of the ACK 80497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @opcode: the opcode of the request that resulted in the ACK 80597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 8066700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * This is called from ipath_rc_rcv_resp() to process an incoming RC ACK 80797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * for the given QP. 80812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * Called at interrupt level with the QP s_lock held and interrupts disabled. 80997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Returns 1 if OK, 0 if current operation should be aborted (NAK). 81097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 81197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanstatic int do_rc_ack(struct ipath_qp *qp, u32 aeth, u32 psn, int opcode) 81297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 81397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_ibdev *dev = to_idev(qp->ibqp.device); 81497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ib_wc wc; 81597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_swqe *wqe; 81697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int ret = 0; 81713b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell u32 ack_psn; 81897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 81997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 82097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Remove the QP from the timeout queue (or RNR timeout queue). 82197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * If ipath_ib_timer() has already removed it, 82297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * it's OK since we hold the QP s_lock and ipath_restart_rc() 82397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * just won't find anything to restart if we ACK everything. 82497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 82597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_lock(&dev->pending_lock); 82694b8d9f98d7f535037eb9845b81396f667b4f727Bryan O'Sullivan if (!list_empty(&qp->timerwait)) 82794b8d9f98d7f535037eb9845b81396f667b4f727Bryan O'Sullivan list_del_init(&qp->timerwait); 82897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_unlock(&dev->pending_lock); 82997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 83097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 83197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Note that NAKs implicitly ACK outstanding SEND and RDMA write 83297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * requests and implicitly NAK RDMA read and atomic requests issued 83397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * before the NAK'ed request. The MSN won't include the NAK'ed 83497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * request but will include an ACK'ed request(s). 83597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 83613b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell ack_psn = psn; 83713b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell if (aeth >> 29) 83813b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell ack_psn--; 83997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe = get_swqe_ptr(qp, qp->s_last); 84097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 84197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 84297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * The MSN might be for a later WQE than the PSN indicates so 84397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * only complete WQEs that the PSN finishes. 84497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 84513b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell while (ipath_cmp24(ack_psn, wqe->lpsn) >= 0) { 84697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 84797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * If this request is a RDMA read or atomic, and the ACK is 84897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * for a later operation, this ACK NAKs the RDMA read or 84997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * atomic. In other words, only a RDMA_READ_LAST or ONLY 85097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * can ACK a RDMA read and likewise for atomic ops. Note 85197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * that the NAK case can only happen if relaxed ordering is 85297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * used and requests are sent after an RDMA read or atomic 85397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * is sent but before the response is received. 85497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 85597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if ((wqe->wr.opcode == IB_WR_RDMA_READ && 85613b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell (opcode != OP(RDMA_READ_RESPONSE_LAST) || 8573859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ipath_cmp24(ack_psn, wqe->lpsn) != 0)) || 85897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 85997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) && 86097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan (opcode != OP(ATOMIC_ACKNOWLEDGE) || 86197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_cmp24(wqe->psn, psn) != 0))) { 86297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 86397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * The last valid PSN seen is the previous 86497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * request's. 86597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 8666022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan update_last_psn(qp, wqe->psn - 1); 86797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Retry this request. */ 86897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_restart_rc(qp, wqe->psn, &wc); 86997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 87097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * No need to process the ACK/NAK since we are 87197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * restarting an earlier request. 87297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 87397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto bail; 87497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 8753859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (qp->s_num_rd_atomic && 8763859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell (wqe->wr.opcode == IB_WR_RDMA_READ || 8773859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 8783859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) { 8793859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_num_rd_atomic--; 8803859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* Restart sending task if fence is complete */ 8813859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if ((qp->s_flags & IPATH_S_FENCE_PENDING) && 8823859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell !qp->s_num_rd_atomic) { 8833859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_flags &= ~IPATH_S_FENCE_PENDING; 8843859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell tasklet_hi_schedule(&qp->s_task); 8853859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } else if (qp->s_flags & IPATH_S_RDMAR_PENDING) { 8863859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_flags &= ~IPATH_S_RDMAR_PENDING; 8873859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell tasklet_hi_schedule(&qp->s_task); 8883859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 8893859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 89097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Post a send completion queue entry if requested. */ 8913859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (!(qp->s_flags & IPATH_S_SIGNAL_REQ_WR) || 89297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan (wqe->wr.send_flags & IB_SEND_SIGNALED)) { 89397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.wr_id = wqe->wr.wr_id; 89497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.status = IB_WC_SUCCESS; 89597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode]; 89697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.vendor_err = 0; 89797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.byte_len = wqe->length; 898062dbb69f32b9ccea701b30f8cc0049482e6211fMichael S. Tsirkin wc.qp = &qp->ibqp; 89997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.src_qp = qp->remote_qpn; 90097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.pkey_index = 0; 90197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.slid = qp->remote_ah_attr.dlid; 90297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.sl = qp->remote_ah_attr.sl; 90397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.dlid_path_bits = 0; 90497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.port_num = 0; 90597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 0); 90697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 90797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_retry = qp->s_retry_cnt; 90897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 90997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * If we are completing a request which is in the process of 91097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * being resent, we can stop resending it since we know the 91197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * responder has already seen it. 91297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 91397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_last == qp->s_cur) { 91497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (++qp->s_cur >= qp->s_size) 91597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur = 0; 91697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe = get_swqe_ptr(qp, qp->s_cur); 91797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(SEND_LAST); 91897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_psn = wqe->psn; 91997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 92097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (++qp->s_last >= qp->s_size) 92197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_last = 0; 92297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe = get_swqe_ptr(qp, qp->s_last); 92397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_last == qp->s_tail) 92497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 92597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 92697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 92797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan switch (aeth >> 29) { 92897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case 0: /* ACK */ 92997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_rc_acks++; 93097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* If this is a partial ACK, reset the retransmit timer. */ 93197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_last != qp->s_tail) { 93297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_lock(&dev->pending_lock); 93397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan list_add_tail(&qp->timerwait, 93497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan &dev->pending[dev->pending_index]); 93597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_unlock(&dev->pending_lock); 93697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 93797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_get_credit(qp, aeth); 93897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_rnr_retry = qp->s_rnr_retry_cnt; 93997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_retry = qp->s_retry_cnt; 9406022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan update_last_psn(qp, psn); 94197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ret = 1; 94297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto bail; 94397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 94497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case 1: /* RNR NAK */ 94597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_rnr_naks++; 94697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_rnr_retry == 0) { 94797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_last == qp->s_tail) 94897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto bail; 94997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 95097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.status = IB_WC_RNR_RETRY_EXC_ERR; 95197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto class_b; 95297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 95397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_rnr_retry_cnt < 7) 95497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_rnr_retry--; 95597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_last == qp->s_tail) 95697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto bail; 95797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 9586700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan /* The last valid PSN is the previous PSN. */ 9596022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan update_last_psn(qp, psn - 1); 96097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 96197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_rc_resends += (int)qp->s_psn - (int)psn; 96297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 9636700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan reset_psn(qp, psn); 96497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 96597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_rnr_timeout = 96627b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan ib_ipath_rnr_table[(aeth >> IPATH_AETH_CREDIT_SHIFT) & 96727b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan IPATH_AETH_CREDIT_MASK]; 96897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_insert_rnr_queue(qp); 96997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto bail; 97097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 97197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case 3: /* NAK */ 97297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* The last valid PSN seen is the previous request's. */ 97397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_last != qp->s_tail) 9746022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan update_last_psn(qp, wqe->psn - 1); 97527b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan switch ((aeth >> IPATH_AETH_CREDIT_SHIFT) & 97627b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan IPATH_AETH_CREDIT_MASK) { 97797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case 0: /* PSN sequence error */ 97897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_seq_naks++; 97997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 98097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Back up to the responder's expected PSN. XXX 98197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Note that we might get a NAK in the middle of an 98297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * RDMA READ response which terminates the RDMA 98397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * READ. 98497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 98597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_last == qp->s_tail) 98697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 98797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 98897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (ipath_cmp24(psn, wqe->psn) < 0) 98997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 99097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 99197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Retry the request. */ 99297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_restart_rc(qp, psn, &wc); 99397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 99497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 99597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case 1: /* Invalid Request */ 99697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.status = IB_WC_REM_INV_REQ_ERR; 99797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_other_naks++; 99897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto class_b; 99997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 100097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case 2: /* Remote Access Error */ 100197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.status = IB_WC_REM_ACCESS_ERR; 100297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_other_naks++; 100397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto class_b; 100497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 100597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case 3: /* Remote Operation Error */ 100697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.status = IB_WC_REM_OP_ERR; 100797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_other_naks++; 100897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan class_b: 100997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.wr_id = wqe->wr.wr_id; 101097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode]; 101197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.vendor_err = 0; 101297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.byte_len = 0; 1013062dbb69f32b9ccea701b30f8cc0049482e6211fMichael S. Tsirkin wc.qp = &qp->ibqp; 101497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.src_qp = qp->remote_qpn; 101597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.pkey_index = 0; 101697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.slid = qp->remote_ah_attr.dlid; 101797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.sl = qp->remote_ah_attr.sl; 101897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.dlid_path_bits = 0; 101997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.port_num = 0; 102097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_sqerror_qp(qp, &wc); 102197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 102297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 102397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan default: 102497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Ignore other reserved NAK error codes */ 102597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto reserved; 102697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 102797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_rnr_retry = qp->s_rnr_retry_cnt; 102897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto bail; 102997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 103097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan default: /* 2: reserved */ 103197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan reserved: 103297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Ignore reserved NAK codes. */ 103397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto bail; 103497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 103597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 103697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanbail: 103797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan return ret; 103897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 103997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 104097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 104197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ipath_rc_rcv_resp - process an incoming RC response packet 104297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @dev: the device this packet came in on 104397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @ohdr: the other headers for this packet 104497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @data: the packet data 104597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @tlen: the packet length 104697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: the QP for this packet 104797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @opcode: the opcode for this packet 104897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @psn: the packet sequence number for this packet 104997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @hdrsize: the header length 105097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @pmtu: the path MTU 105197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @header_in_data: true if part of the header data is in the data buffer 105297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 105397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * This is called from ipath_rc_rcv() to process an incoming RC response 105497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * packet for the given QP. 105597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Called at interrupt level. 105697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 105797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanstatic inline void ipath_rc_rcv_resp(struct ipath_ibdev *dev, 105897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_other_headers *ohdr, 105997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan void *data, u32 tlen, 106097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_qp *qp, 106197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 opcode, 106297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 psn, u32 hdrsize, u32 pmtu, 106397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int header_in_data) 106497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 10653859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell struct ipath_swqe *wqe; 106697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan unsigned long flags; 106797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ib_wc wc; 106897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int diff; 106997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 pad; 107097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 aeth; 107197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 107297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_lock_irqsave(&qp->s_lock, flags); 107397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 107497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Ignore invalid responses. */ 107597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (ipath_cmp24(psn, qp->s_next_psn) >= 0) 107697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto ack_done; 107797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 107897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Ignore duplicate responses. */ 107997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan diff = ipath_cmp24(psn, qp->s_last_psn); 108097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(diff <= 0)) { 108197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Update credits for "ghost" ACKs */ 108297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (diff == 0 && opcode == OP(ACKNOWLEDGE)) { 108397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!header_in_data) 108497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan aeth = be32_to_cpu(ohdr->u.aeth); 108597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 108697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan aeth = be32_to_cpu(((__be32 *) data)[0]); 108797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(__be32); 108897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 108997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if ((aeth >> 29) == 0) 109097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_get_credit(qp, aeth); 109197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 109297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto ack_done; 109397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 109497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 10953859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(qp->s_last == qp->s_tail)) 10963859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto ack_done; 10973859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe = get_swqe_ptr(qp, qp->s_last); 10983859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 109997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan switch (opcode) { 110097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(ACKNOWLEDGE): 110197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(ATOMIC_ACKNOWLEDGE): 110297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_FIRST): 110397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!header_in_data) 110497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan aeth = be32_to_cpu(ohdr->u.aeth); 110597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 110697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan aeth = be32_to_cpu(((__be32 *) data)[0]); 110797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(__be32); 110897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 11093859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (opcode == OP(ATOMIC_ACKNOWLEDGE)) { 11103859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u64 val; 11113859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 11123859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (!header_in_data) { 11133859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell __be32 *p = ohdr->u.at.atomic_ack_eth; 11143859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 11153859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell val = ((u64) be32_to_cpu(p[0]) << 32) | 11163859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell be32_to_cpu(p[1]); 11173859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } else 11183859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell val = be64_to_cpu(((__be64 *) data)[0]); 11193859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell *(u64 *) wqe->sg_list[0].vaddr = val; 11203859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 112197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!do_rc_ack(qp, aeth, psn, opcode) || 112297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode != OP(RDMA_READ_RESPONSE_FIRST)) 112397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto ack_done; 112497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hdrsize += 4; 11253859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 11263859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto ack_done; 112797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 11283859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * If this is a response to a resent RDMA read, we 11293859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * have to be careful to copy the data to the right 11303859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * location. 113197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 11323859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, 11333859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe, psn, pmtu); 11343859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto read_middle; 113597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 113697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_MIDDLE): 113797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* no AETH, no ACK */ 113897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(ipath_cmp24(psn, qp->s_last_psn + 1))) { 113997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_rdma_seq++; 11403859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ipath_restart_rc(qp, qp->s_last_psn + 1, &wc); 114197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto ack_done; 114297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 11433859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 11447bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan goto ack_done; 11453859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell read_middle: 11467bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan if (unlikely(tlen != (hdrsize + pmtu + 4))) 11477bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan goto ack_done; 11483859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(pmtu >= qp->s_rdma_read_len)) 11497bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan goto ack_done; 11503859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 11517bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan /* We got a response so update the timeout. */ 11527bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan spin_lock(&dev->pending_lock); 11537bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan if (qp->s_rnr_timeout == 0 && !list_empty(&qp->timerwait)) 11547bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan list_move_tail(&qp->timerwait, 11557bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan &dev->pending[dev->pending_index]); 11567bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan spin_unlock(&dev->pending_lock); 11577bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan /* 11586700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * Update the RDMA receive state but do the copy w/o 11596700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * holding the locks and blocking interrupts. 11607bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan */ 11613859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_rdma_read_len -= pmtu; 11626022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan update_last_psn(qp, psn); 11637bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan spin_unlock_irqrestore(&qp->s_lock, flags); 11643859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ipath_copy_sge(&qp->s_rdma_read_sge, data, pmtu); 11657bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan goto bail; 116697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 11673859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(RDMA_READ_RESPONSE_ONLY): 11683859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(ipath_cmp24(psn, qp->s_last_psn + 1))) { 11693859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell dev->n_rdma_seq++; 11703859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ipath_restart_rc(qp, qp->s_last_psn + 1, &wc); 11713859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto ack_done; 11723859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 11733859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 11743859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto ack_done; 11753859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 11763859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * If this is a response to a resent RDMA read, we 11773859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * have to be careful to copy the data to the right 11783859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * location. 11793859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * XXX should check PSN and wqe opcode first. 11803859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 11813859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, 11823859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe, psn, pmtu); 11833859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto read_last; 11843859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 118597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_LAST): 118697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* ACKs READ req. */ 118797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(ipath_cmp24(psn, qp->s_last_psn + 1))) { 118897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_rdma_seq++; 11893859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ipath_restart_rc(qp, qp->s_last_psn + 1, &wc); 119097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto ack_done; 119197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 11923859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 119397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto ack_done; 11943859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell read_last: 119597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 119697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Get the number of bytes the message was padded by. 119797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 119897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 119997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 120097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Check that the data size is >= 1 && <= pmtu. 120197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Remember to account for the AETH header (4) and 120297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ICRC (4). 120397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 120497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(tlen <= (hdrsize + pad + 8))) { 1205ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan /* XXX Need to generate an error CQ entry. */ 120697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto ack_done; 120797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 120897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan tlen -= hdrsize + pad + 8; 12093859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(tlen != qp->s_rdma_read_len)) { 1210ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan /* XXX Need to generate an error CQ entry. */ 121197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto ack_done; 121297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 121397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!header_in_data) 121497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan aeth = be32_to_cpu(ohdr->u.aeth); 121597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 121697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan aeth = be32_to_cpu(((__be32 *) data)[0]); 121797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(__be32); 121897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 12193859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ipath_copy_sge(&qp->s_rdma_read_sge, data, tlen); 12203859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell (void) do_rc_ack(qp, aeth, psn, OP(RDMA_READ_RESPONSE_LAST)); 122197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto ack_done; 122297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 122397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 122497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanack_done: 122597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_unlock_irqrestore(&qp->s_lock, flags); 122697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanbail: 122797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan return; 122897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 122997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 123097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 123197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ipath_rc_rcv_error - process an incoming duplicate or error RC packet 123297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @dev: the device this packet came in on 123397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @ohdr: the other headers for this packet 123497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @data: the packet data 123597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: the QP for this packet 123697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @opcode: the opcode for this packet 123797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @psn: the packet sequence number for this packet 123897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @diff: the difference between the PSN and the expected PSN 123997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @header_in_data: true if part of the header data is in the data buffer 124097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 124197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * This is called from ipath_rc_rcv() to process an unexpected 124297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * incoming RC packet for the given QP. 124397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Called at interrupt level. 124497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Return 1 if no more processing is needed; otherwise return 0 to 12453859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * schedule a response to be sent. 124697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 124797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanstatic inline int ipath_rc_rcv_error(struct ipath_ibdev *dev, 124897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_other_headers *ohdr, 124997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan void *data, 125097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_qp *qp, 125197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 opcode, 125297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 psn, 125397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int diff, 125497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int header_in_data) 125597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 12563859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell struct ipath_ack_entry *e; 12573859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u8 i, prev; 12583859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell int old_req; 125997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 126097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (diff > 0) { 126197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 126297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Packet sequence error. 126397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * A NAK will ACK earlier sends and RDMA writes. 12643859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Don't queue the NAK if we already sent one. 126597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 12663859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (!qp->r_nak_state) { 126712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_nak_state = IB_NAK_PSN_ERROR; 126812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan /* Use the expected PSN. */ 126912eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_ack_psn = qp->r_psn; 12703859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto send_ack; 127197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 12723859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto done; 127397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 127497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 127597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 127697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Handle a duplicate request. Don't re-execute SEND, RDMA 127797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * write or atomic op. Don't NAK errors, just silently drop 127897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * the duplicate request. Note that r_sge, r_len, and 127997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * r_rcv_len may be in use so don't modify them. 128097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 128197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * We are supposed to ACK the earliest duplicate PSN but we 128297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * can coalesce an outstanding duplicate ACK. We have to 128397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * send the earliest so that RDMA reads can be restarted at 128497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * the requester's expected PSN. 12853859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * 12863859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * First, find where this duplicate PSN falls within the 12873859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * ACKs previously sent. 128897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 12893859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell psn &= IPATH_PSN_MASK; 12903859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e = NULL; 12913859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell old_req = 1; 12923859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell spin_lock_irq(&qp->s_lock); 12933859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell for (i = qp->r_head_ack_queue; ; i = prev) { 12943859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (i == qp->s_tail_ack_queue) 12953859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell old_req = 0; 12963859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (i) 12973859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell prev = i - 1; 12983859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell else 12993859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell prev = IPATH_MAX_RDMA_ATOMIC; 13003859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (prev == qp->r_head_ack_queue) { 13013859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e = NULL; 13023859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell break; 13033859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 13043859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e = &qp->s_ack_queue[prev]; 13053859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (!e->opcode) { 13063859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e = NULL; 13073859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell break; 13083859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 13093859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (ipath_cmp24(psn, e->psn) >= 0) 13103859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell break; 13113859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 13123859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell switch (opcode) { 13133859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(RDMA_READ_REQUEST): { 13143859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell struct ib_reth *reth; 13153859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u32 offset; 13163859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u32 len; 13173859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 13183859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 13193859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * If we didn't find the RDMA read request in the ack queue, 13203859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * or the send tasklet is already backed up to send an 13213859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * earlier entry, we can ignore this request. 13223859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 13233859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (!e || e->opcode != OP(RDMA_READ_REQUEST) || old_req) 13243859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto unlock_done; 132597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* RETH comes after BTH */ 132697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!header_in_data) 132797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan reth = &ohdr->u.rc.reth; 132897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 132997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan reth = (struct ib_reth *)data; 133097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(*reth); 133197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 133212eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan /* 13333859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Address range must be a subset of the original 13343859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * request and start on pmtu boundaries. 13353859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * We reuse the old ack_queue slot since the requester 13363859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * should not back up and request an earlier PSN for the 13373859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * same request. 133812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan */ 13393859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell offset = ((psn - e->psn) & IPATH_PSN_MASK) * 13403859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ib_mtu_enum_to_int(qp->path_mtu); 13413859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = be32_to_cpu(reth->length); 13423859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(offset + len > e->rdma_sge.sge.sge_length)) 13433859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto unlock_done; 13443859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (len != 0) { 134597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 rkey = be32_to_cpu(reth->rkey); 134697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u64 vaddr = be64_to_cpu(reth->vaddr); 134797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int ok; 134897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 13493859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ok = ipath_rkey_ok(qp, &e->rdma_sge, 13503859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len, vaddr, rkey, 135197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan IB_ACCESS_REMOTE_READ); 13523859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(!ok)) 13533859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto unlock_done; 135497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } else { 13553859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sg_list = NULL; 13563859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.num_sge = 0; 13573859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.mr = NULL; 13583859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.vaddr = NULL; 13593859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.length = 0; 13603859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.sge_length = 0; 136197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 13623859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->psn = psn; 13633859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(ACKNOWLEDGE); 13643859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_tail_ack_queue = prev; 13653859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell break; 136612eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan } 136712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan 136897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(COMPARE_SWAP): 13693859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(FETCH_ADD): { 137097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 13713859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * If we didn't find the atomic request in the ack queue 13723859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * or the send tasklet is already backed up to send an 13733859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * earlier entry, we can ignore this request. 137497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 13753859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (!e || e->opcode != (u8) opcode || old_req) 13763859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto unlock_done; 13773859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(ACKNOWLEDGE); 13783859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_tail_ack_queue = prev; 13793859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell break; 13803859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 13813859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 13823859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell default: 13833859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (old_req) 13843859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto unlock_done; 13853859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 13863859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Resend the most recent ACK if this request is 13873859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * after all the previous RDMA reads and atomics. 13883859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 13893859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (i == qp->r_head_ack_queue) { 13903859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell spin_unlock_irq(&qp->s_lock); 13913859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_nak_state = 0; 13923859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_ack_psn = qp->r_psn - 1; 13933859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto send_ack; 13943859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 13953859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 13963859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Resend the RDMA read or atomic op which 13973859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * ACKs this duplicate request. 13983859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 13993859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(ACKNOWLEDGE); 14003859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_tail_ack_queue = i; 140197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 140297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 140312eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_nak_state = 0; 14043859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell spin_unlock_irq(&qp->s_lock); 14053859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell tasklet_hi_schedule(&qp->s_task); 140697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 14073859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbellunlock_done: 14083859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell spin_unlock_irq(&qp->s_lock); 140997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivandone: 141097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan return 1; 14113859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 14123859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbellsend_ack: 14133859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell return 0; 141497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 141597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 14168d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivanstatic void ipath_rc_error(struct ipath_qp *qp, enum ib_wc_status err) 14178d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivan{ 14188d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivan spin_lock_irq(&qp->s_lock); 14198d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivan qp->state = IB_QPS_ERR; 14208d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivan ipath_error_qp(qp, err); 14218d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivan spin_unlock_irq(&qp->s_lock); 14228d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivan} 14238d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivan 142497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 142597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ipath_rc_rcv - process an incoming RC packet 142697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @dev: the device this packet came in on 142797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @hdr: the header of this packet 142897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @has_grh: true if the header has a GRH 142997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @data: the packet data 143097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @tlen: the packet length 143197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: the QP for this packet 143297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 143397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * This is called from ipath_qp_rcv() to process an incoming RC packet 143497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * for the given QP. 143597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Called at interrupt level. 143697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 143797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanvoid ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 143897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int has_grh, void *data, u32 tlen, struct ipath_qp *qp) 143997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 144097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_other_headers *ohdr; 144197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 opcode; 144297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 hdrsize; 144397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 psn; 144497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 pad; 144597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ib_wc wc; 144697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu); 144797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int diff; 144897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ib_reth *reth; 144997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int header_in_data; 145097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 145110aeb0e6d8823c1cccf9edc8401c848745c128beBryan O'Sullivan /* Validate the SLID. See Ch. 9.6.1.5 */ 145210aeb0e6d8823c1cccf9edc8401c848745c128beBryan O'Sullivan if (unlikely(be16_to_cpu(hdr->lrh[3]) != qp->remote_ah_attr.dlid)) 145310aeb0e6d8823c1cccf9edc8401c848745c128beBryan O'Sullivan goto done; 145410aeb0e6d8823c1cccf9edc8401c848745c128beBryan O'Sullivan 145597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check for GRH */ 145697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!has_grh) { 145797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr = &hdr->u.oth; 145897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hdrsize = 8 + 12; /* LRH + BTH */ 145997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan psn = be32_to_cpu(ohdr->bth[2]); 146097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan header_in_data = 0; 146197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } else { 146297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr = &hdr->u.l.oth; 146397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hdrsize = 8 + 40 + 12; /* LRH + GRH + BTH */ 146497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 146597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * The header with GRH is 60 bytes and the core driver sets 146697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * the eager header buffer size to 56 bytes so the last 4 146797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * bytes of the BTH header (PSN) is in the data buffer. 146897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 146934b2aafea38efdf02cd8107a6e1057e2a297c447Bryan O'Sullivan header_in_data = dev->dd->ipath_rcvhdrentsize == 16; 147097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (header_in_data) { 147197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan psn = be32_to_cpu(((__be32 *) data)[0]); 147297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(__be32); 147397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } else 147497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan psn = be32_to_cpu(ohdr->bth[2]); 147597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 147697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 147797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 147897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Process responses (ACKs) before anything else. Note that the 147997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * packet sequence number will be for something in the send work 148097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * queue rather than the expected receive packet sequence number. 148197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * In other words, this QP is the requester. 148297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 1483ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan opcode = be32_to_cpu(ohdr->bth[0]) >> 24; 148497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) && 148597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode <= OP(ATOMIC_ACKNOWLEDGE)) { 148697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_rc_rcv_resp(dev, ohdr, data, tlen, qp, opcode, psn, 148797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hdrsize, pmtu, header_in_data); 148812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto done; 148997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 149097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 149197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Compute 24 bits worth of difference. */ 149297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan diff = ipath_cmp24(psn, qp->r_psn); 149397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(diff)) { 149497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (ipath_rc_rcv_error(dev, ohdr, data, qp, opcode, 149597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan psn, diff, header_in_data)) 149697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto done; 149712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto send_ack; 149897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 149997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 150097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check for opcode sequence errors. */ 150197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan switch (qp->r_state) { 150297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_FIRST): 150397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_MIDDLE): 150497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (opcode == OP(SEND_MIDDLE) || 150597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(SEND_LAST) || 150697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(SEND_LAST_WITH_IMMEDIATE)) 150797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 150897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan nack_inv: 15098d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivan ipath_rc_error(qp, IB_WC_REM_INV_REQ_ERR); 151012eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_nak_state = IB_NAK_INVALID_REQUEST; 151112eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_ack_psn = qp->r_psn; 151212eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto send_ack; 151397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 151497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_FIRST): 151597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_MIDDLE): 151697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (opcode == OP(RDMA_WRITE_MIDDLE) || 151797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(RDMA_WRITE_LAST) || 151897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) 151997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 152097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_inv; 152197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 152297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan default: 152397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (opcode == OP(SEND_MIDDLE) || 152497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(SEND_LAST) || 152597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(SEND_LAST_WITH_IMMEDIATE) || 152697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(RDMA_WRITE_MIDDLE) || 152797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(RDMA_WRITE_LAST) || 152897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) 152997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_inv; 153012eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan /* 153112eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * Note that it is up to the requester to not send a new 153212eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * RDMA read or atomic operation before receiving an ACK 153312eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * for the previous operation. 153412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan */ 153597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 153697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 153797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 153897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.imm_data = 0; 153997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.wc_flags = 0; 154097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 154197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* OK, process the packet. */ 154297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan switch (opcode) { 154397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_FIRST): 154497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!ipath_get_rwqe(qp, 0)) { 154597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan rnr_nak: 154697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 154797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * A RNR NAK will ACK earlier sends and RDMA writes. 154897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Don't queue the NAK if a RDMA read or atomic 154997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * is pending though. 155097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 15513859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (qp->r_nak_state) 15523859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto done; 155312eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_nak_state = IB_RNR_NAK | qp->r_min_rnr_timer; 155412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_ack_psn = qp->r_psn; 155512eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto send_ack; 155697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 155797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_rcv_len = 0; 155897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 155997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_MIDDLE): 156097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_MIDDLE): 156197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan send_middle: 156297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check for invalid length PMTU or posted rwqe len. */ 156397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(tlen != (hdrsize + pmtu + 4))) 156497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_inv; 156597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_rcv_len += pmtu; 156697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(qp->r_rcv_len > qp->r_len)) 156797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_inv; 156897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_copy_sge(&qp->r_sge, data, pmtu); 156997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 157097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 157197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE): 157297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* consume RWQE */ 157397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!ipath_get_rwqe(qp, 1)) 157497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto rnr_nak; 157597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto send_last_imm; 157697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 157797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_ONLY): 157897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_ONLY_WITH_IMMEDIATE): 157997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!ipath_get_rwqe(qp, 0)) 158097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto rnr_nak; 158197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_rcv_len = 0; 158297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (opcode == OP(SEND_ONLY)) 158397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto send_last; 158497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 158597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_LAST_WITH_IMMEDIATE): 158697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan send_last_imm: 158797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (header_in_data) { 158897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.imm_data = *(__be32 *) data; 158997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(__be32); 159097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } else { 159197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Immediate data comes after BTH */ 159297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.imm_data = ohdr->u.imm_data; 159397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 159497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hdrsize += 4; 159597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.wc_flags = IB_WC_WITH_IMM; 159697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 159797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_LAST): 159897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_LAST): 159997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan send_last: 160097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Get the number of bytes the message was padded by. */ 160197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 160297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check for invalid length. */ 160397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* XXX LAST len should be >= 1 */ 160497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(tlen < (hdrsize + pad + 4))) 160597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_inv; 160697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Don't count the CRC. */ 160797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan tlen -= (hdrsize + pad + 4); 160897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.byte_len = tlen + qp->r_rcv_len; 160997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(wc.byte_len > qp->r_len)) 161097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_inv; 161197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_copy_sge(&qp->r_sge, data, tlen); 161212eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_msn++; 16138d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivan if (!qp->r_wrid_valid) 161497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 16158d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivan qp->r_wrid_valid = 0; 161697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.wr_id = qp->r_wr_id; 161797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.status = IB_WC_SUCCESS; 161897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.opcode = IB_WC_RECV; 161997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.vendor_err = 0; 1620062dbb69f32b9ccea701b30f8cc0049482e6211fMichael S. Tsirkin wc.qp = &qp->ibqp; 162197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.src_qp = qp->remote_qpn; 162297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.pkey_index = 0; 162397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.slid = qp->remote_ah_attr.dlid; 162497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.sl = qp->remote_ah_attr.sl; 162597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.dlid_path_bits = 0; 162697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.port_num = 0; 162797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Signal completion event if the solicited bit is set. */ 162897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 162997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan (ohdr->bth[0] & 163097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan __constant_cpu_to_be32(1 << 23)) != 0); 163197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 163297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 163397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_FIRST): 163497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_ONLY): 163597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): 163697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* consume RWQE */ 163797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* RETH comes after BTH */ 163897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!header_in_data) 163997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan reth = &ohdr->u.rc.reth; 164097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 164197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan reth = (struct ib_reth *)data; 164297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(*reth); 164397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 164497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hdrsize += sizeof(*reth); 164597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_len = be32_to_cpu(reth->length); 164697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_rcv_len = 0; 164797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->r_len != 0) { 164897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 rkey = be32_to_cpu(reth->rkey); 164997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u64 vaddr = be64_to_cpu(reth->vaddr); 165097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int ok; 165197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 165297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check rkey & NAK */ 16536a553af286653818bb5831f1b351eefdc8a93b61Bryan O'Sullivan ok = ipath_rkey_ok(qp, &qp->r_sge, 165497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_len, vaddr, rkey, 165597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan IB_ACCESS_REMOTE_WRITE); 165612eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan if (unlikely(!ok)) 165712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto nack_acc; 165897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } else { 165997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_sge.sg_list = NULL; 166097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_sge.sge.mr = NULL; 166197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_sge.sge.vaddr = NULL; 166297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_sge.sge.length = 0; 166397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_sge.sge.sge_length = 0; 166497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 166597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(!(qp->qp_access_flags & 166697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan IB_ACCESS_REMOTE_WRITE))) 166797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_acc; 166897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (opcode == OP(RDMA_WRITE_FIRST)) 166997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto send_middle; 167097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else if (opcode == OP(RDMA_WRITE_ONLY)) 167197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto send_last; 167297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!ipath_get_rwqe(qp, 1)) 167397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto rnr_nak; 167497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto send_last_imm; 167597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 16763859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(RDMA_READ_REQUEST): { 16773859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell struct ipath_ack_entry *e; 16783859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u32 len; 16793859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u8 next; 16803859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 16813859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) 16823859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto nack_acc; 16833859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell next = qp->r_head_ack_queue + 1; 16843859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (next > IPATH_MAX_RDMA_ATOMIC) 16853859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell next = 0; 16863859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(next == qp->s_tail_ack_queue)) 16873859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto nack_inv; 16883859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e = &qp->s_ack_queue[qp->r_head_ack_queue]; 168997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* RETH comes after BTH */ 169097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!header_in_data) 169197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan reth = &ohdr->u.rc.reth; 169297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 169397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan reth = (struct ib_reth *)data; 169497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(*reth); 169597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 16963859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = be32_to_cpu(reth->length); 16973859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (len) { 169897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 rkey = be32_to_cpu(reth->rkey); 169997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u64 vaddr = be64_to_cpu(reth->vaddr); 170097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int ok; 170197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 170297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check rkey & NAK */ 17033859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ok = ipath_rkey_ok(qp, &e->rdma_sge, len, vaddr, 17043859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell rkey, IB_ACCESS_REMOTE_READ); 17053859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(!ok)) 170697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_acc; 170797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 170897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Update the next expected PSN. We add 1 later 170997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * below, so only add the remainder here. 171097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 17113859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (len > pmtu) 17123859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_psn += (len - 1) / pmtu; 171397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } else { 17143859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sg_list = NULL; 17153859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.num_sge = 0; 17163859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.mr = NULL; 17173859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.vaddr = NULL; 17183859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.length = 0; 17193859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.sge_length = 0; 172097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 17213859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->opcode = opcode; 17223859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->psn = psn; 172397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 172497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * We need to increment the MSN here instead of when we 172597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * finish sending the result since a duplicate request would 172697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * increment it more than once. 172797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 172812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_msn++; 172997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_psn++; 173097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_state = opcode; 173112eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_nak_state = 0; 17323859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell barrier(); 17333859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_head_ack_queue = next; 173412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan 173512eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan /* Call ipath_do_rc_send() in another thread. */ 173612eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan tasklet_hi_schedule(&qp->s_task); 173712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan 173812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto done; 17393859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 174097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 174197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(COMPARE_SWAP): 174297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(FETCH_ADD): { 174397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ib_atomic_eth *ateth; 17443859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell struct ipath_ack_entry *e; 174597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u64 vaddr; 17463859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell atomic64_t *maddr; 174797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u64 sdata; 174897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 rkey; 17493859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u8 next; 175097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 17513859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(!(qp->qp_access_flags & 17523859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell IB_ACCESS_REMOTE_ATOMIC))) 17533859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto nack_acc; 17543859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell next = qp->r_head_ack_queue + 1; 17553859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (next > IPATH_MAX_RDMA_ATOMIC) 17563859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell next = 0; 17573859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(next == qp->s_tail_ack_queue)) 17583859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto nack_inv; 175997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!header_in_data) 176097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ateth = &ohdr->u.atomic_eth; 17613859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell else 176297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ateth = (struct ib_atomic_eth *)data; 17633859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell vaddr = ((u64) be32_to_cpu(ateth->vaddr[0]) << 32) | 17643859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell be32_to_cpu(ateth->vaddr[1]); 176597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(vaddr & (sizeof(u64) - 1))) 176697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_inv; 176797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan rkey = be32_to_cpu(ateth->rkey); 176897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check rkey & NAK */ 17696a553af286653818bb5831f1b351eefdc8a93b61Bryan O'Sullivan if (unlikely(!ipath_rkey_ok(qp, &qp->r_sge, 177097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan sizeof(u64), vaddr, rkey, 177197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan IB_ACCESS_REMOTE_ATOMIC))) 177297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_acc; 177397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Perform atomic OP and save result. */ 17743859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell maddr = (atomic64_t *) qp->r_sge.sge.vaddr; 177597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan sdata = be64_to_cpu(ateth->swap_data); 17763859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e = &qp->s_ack_queue[qp->r_head_ack_queue]; 17773859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->atomic_data = (opcode == OP(FETCH_ADD)) ? 17783859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell (u64) atomic64_add_return(sdata, maddr) - sdata : 17793859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr, 17803859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell be64_to_cpu(ateth->compare_data), 17813859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell sdata); 17823859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->opcode = opcode; 17833859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->psn = psn & IPATH_PSN_MASK; 178412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_msn++; 17853859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_psn++; 17863859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_state = opcode; 17873859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_nak_state = 0; 17883859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell barrier(); 17893859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_head_ack_queue = next; 17903859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 17913859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* Call ipath_do_rc_send() in another thread. */ 17923859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell tasklet_hi_schedule(&qp->s_task); 17933859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 17943859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto done; 179597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 179697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 179797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan default: 17983859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* NAK unknown opcodes. */ 17993859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto nack_inv; 180097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 180197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_psn++; 180297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_state = opcode; 18033859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_ack_psn = psn; 180412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_nak_state = 0; 180597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Send an ACK if requested or required. */ 18063859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (psn & (1 << 31)) 180712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto send_ack; 180812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto done; 180997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 181012eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivannack_acc: 18113859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ipath_rc_error(qp, IB_WC_REM_ACCESS_ERR); 18123859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; 18133859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_ack_psn = qp->r_psn; 18143859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 181512eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivansend_ack: 18163859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell send_rc_ack(qp); 181797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 181812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivandone: 181997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan return; 182097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 1821