197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/* 2e7eacd36865ae0707f5efae8e4dda421ffcd1b66Ralph Campbell * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved. 397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. 497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * This software is available to you under a choice of one of two 697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * licenses. You may choose to be licensed under the terms of the GNU 797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * General Public License (GPL) Version 2, available from the file 897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * COPYING in the main directory of this source tree, or the 997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * OpenIB.org BSD license below: 1097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 1197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Redistribution and use in source and binary forms, with or 1297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * without modification, are permitted provided that the following 1397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * conditions are met: 1497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 1597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * - Redistributions of source code must retain the above 1697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * copyright notice, this list of conditions and the following 1797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * disclaimer. 1897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 1997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * - Redistributions in binary form must reproduce the above 2097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * copyright notice, this list of conditions and the following 2197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * disclaimer in the documentation and/or other materials 2297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * provided with the distribution. 2397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 2497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 2597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 2697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 2797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 2897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 2997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 3097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 3197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * SOFTWARE. 3297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 3397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 34d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell#include <linux/io.h> 35d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell 3697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan#include "ipath_verbs.h" 3734b2aafea38efdf02cd8107a6e1057e2a297c447Bryan O'Sullivan#include "ipath_kernel.h" 3897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 3997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/* cut down ridiculously long IB macro names */ 4097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan#define OP(x) IB_OPCODE_RC_##x 4197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 423859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbellstatic u32 restart_sge(struct ipath_sge_state *ss, struct ipath_swqe *wqe, 433859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u32 psn, u32 pmtu) 443859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell{ 453859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u32 len; 463859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 473859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = ((psn - wqe->psn) & IPATH_PSN_MASK) * pmtu; 483859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ss->sge = wqe->sg_list[0]; 493859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ss->sg_list = wqe->sg_list + 1; 503859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ss->num_sge = wqe->wr.num_sge; 513859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ipath_skip_sge(ss, len); 523859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell return wqe->length - len; 533859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell} 543859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 5597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 5697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ipath_init_restart- initialize the qp->s_sge after a restart 5797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: the QP who's SGE we're restarting 5897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @wqe: the work queue to initialize the QP's SGE from 5997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 6012eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * The QP s_lock should be held and interrupts disabled. 6197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 6297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanstatic void ipath_init_restart(struct ipath_qp *qp, struct ipath_swqe *wqe) 6397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 6497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_ibdev *dev; 6597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 663859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, 673859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ib_mtu_enum_to_int(qp->path_mtu)); 6897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev = to_idev(qp->ibqp.device); 6997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_lock(&dev->pending_lock); 7094b8d9f98d7f535037eb9845b81396f667b4f727Bryan O'Sullivan if (list_empty(&qp->timerwait)) 7197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan list_add_tail(&qp->timerwait, 7297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan &dev->pending[dev->pending_index]); 7397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_unlock(&dev->pending_lock); 7497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 7597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 7697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 7797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ipath_make_rc_ack - construct a response packet (ACK, NAK, or RDMA read) 7897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: a pointer to the QP 7997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @ohdr: a pointer to the IB header being constructed 8097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @pmtu: the path MTU 8197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 823859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Return 1 if constructed; otherwise, return 0. 833859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Note that we are in the responder's side of the QP context. 8497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Note the QP s_lock must be held. 8597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 864ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbellstatic int ipath_make_rc_ack(struct ipath_ibdev *dev, struct ipath_qp *qp, 874ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell struct ipath_other_headers *ohdr, u32 pmtu) 8897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 893859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell struct ipath_ack_entry *e; 9097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 hwords; 9197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 len; 9297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 bth0; 933859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u32 bth2; 9497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 95e509be898d8937634437caa474b57ac12795e5bcRalph Campbell /* Don't send an ACK if we aren't supposed to. */ 96e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) 97e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto bail; 98e509be898d8937634437caa474b57ac12795e5bcRalph Campbell 9997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* header size in 32-bit words LRH+BTH = (8+12)/4. */ 10097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords = 5; 10197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 10297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan switch (qp->s_ack_state) { 1033859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(RDMA_READ_RESPONSE_LAST): 1043859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(RDMA_READ_RESPONSE_ONLY): 1053859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(ATOMIC_ACKNOWLEDGE): 106154257f3626ea6dd96781fac0896c3f27fe2b0a1Ralph Campbell /* 107154257f3626ea6dd96781fac0896c3f27fe2b0a1Ralph Campbell * We can increment the tail pointer now that the last 108154257f3626ea6dd96781fac0896c3f27fe2b0a1Ralph Campbell * response has been sent instead of only being 109154257f3626ea6dd96781fac0896c3f27fe2b0a1Ralph Campbell * constructed. 110154257f3626ea6dd96781fac0896c3f27fe2b0a1Ralph Campbell */ 111154257f3626ea6dd96781fac0896c3f27fe2b0a1Ralph Campbell if (++qp->s_tail_ack_queue > IPATH_MAX_RDMA_ATOMIC) 112154257f3626ea6dd96781fac0896c3f27fe2b0a1Ralph Campbell qp->s_tail_ack_queue = 0; 1133859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* FALLTHROUGH */ 114154257f3626ea6dd96781fac0896c3f27fe2b0a1Ralph Campbell case OP(SEND_ONLY): 1153859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(ACKNOWLEDGE): 1163859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* Check for no next entry in the queue. */ 1173859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (qp->r_head_ack_queue == qp->s_tail_ack_queue) { 1183859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (qp->s_flags & IPATH_S_ACK_PENDING) 1193859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto normal; 120154257f3626ea6dd96781fac0896c3f27fe2b0a1Ralph Campbell qp->s_ack_state = OP(ACKNOWLEDGE); 1213859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 1223859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 1233859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 1243859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e = &qp->s_ack_queue[qp->s_tail_ack_queue]; 1253859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (e->opcode == OP(RDMA_READ_REQUEST)) { 1263859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* Copy SGE state in case we need to resend */ 1273859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_rdma_sge = e->rdma_sge; 1283859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_cur_sge = &qp->s_ack_rdma_sge; 1293859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = e->rdma_sge.sge.sge_length; 1303859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (len > pmtu) { 1313859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = pmtu; 1323859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST); 133d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell } else { 1343859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY); 135d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell e->sent = 1; 136d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell } 1373859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.aeth = ipath_compute_aeth(qp); 1383859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell hwords++; 1393859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_rdma_psn = e->psn; 1403859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell bth2 = qp->s_ack_rdma_psn++ & IPATH_PSN_MASK; 1413859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } else { 1423859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* COMPARE_SWAP or FETCH_ADD */ 1433859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_cur_sge = NULL; 1443859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = 0; 1453859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); 1463859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.at.aeth = ipath_compute_aeth(qp); 1473859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.at.atomic_ack_eth[0] = 1483859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell cpu_to_be32(e->atomic_data >> 32); 1493859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.at.atomic_ack_eth[1] = 1503859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell cpu_to_be32(e->atomic_data); 1513859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell hwords += sizeof(ohdr->u.at) / sizeof(u32); 1523859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell bth2 = e->psn; 153d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell e->sent = 1; 1543859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 15597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth0 = qp->s_ack_state << 24; 15697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 15797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 15897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_FIRST): 15997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE); 16097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 16197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_MIDDLE): 1623859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = qp->s_ack_rdma_sge.sge.sge_length; 16397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (len > pmtu) 16497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = pmtu; 16597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 16697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.aeth = ipath_compute_aeth(qp); 16797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords++; 16897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST); 169d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell qp->s_ack_queue[qp->s_tail_ack_queue].sent = 1; 17097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 17197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth0 = qp->s_ack_state << 24; 1723859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell bth2 = qp->s_ack_rdma_psn++ & IPATH_PSN_MASK; 17397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 17497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 17597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan default: 1763859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell normal: 17712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan /* 1783859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Send a regular ACK. 1793859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Set the s_ack_state so we wait until after sending 1803859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * the ACK before setting s_ack_state to ACKNOWLEDGE 1813859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * (see above). 18212eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan */ 183154257f3626ea6dd96781fac0896c3f27fe2b0a1Ralph Campbell qp->s_ack_state = OP(SEND_ONLY); 1843859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_flags &= ~IPATH_S_ACK_PENDING; 1853859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_cur_sge = NULL; 18612eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan if (qp->s_nak_state) 1873859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.aeth = 1883859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell cpu_to_be32((qp->r_msn & IPATH_MSN_MASK) | 1893859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell (qp->s_nak_state << 1903859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell IPATH_AETH_CREDIT_SHIFT)); 19112eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan else 19212eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan ohdr->u.aeth = ipath_compute_aeth(qp); 19397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords++; 1943859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = 0; 1953859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell bth0 = OP(ACKNOWLEDGE) << 24; 1963859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell bth2 = qp->s_ack_psn & IPATH_PSN_MASK; 19797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 19897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_hdrwords = hwords; 19997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur_size = len; 2004ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell ipath_make_ruc_header(dev, qp, ohdr, bth0, bth2); 2013859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell return 1; 20297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 20312eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivanbail: 2043859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell return 0; 20597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 20697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 20797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 20897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ipath_make_rc_req - construct a request packet (SEND, RDMA r/w, ATOMIC) 20997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: a pointer to the QP 21097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 21197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Return 1 if constructed; otherwise, return 0. 21297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 2134ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbellint ipath_make_rc_req(struct ipath_qp *qp) 21497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 21597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_ibdev *dev = to_idev(qp->ibqp.device); 2164ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell struct ipath_other_headers *ohdr; 21797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_sge_state *ss; 21897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_swqe *wqe; 21997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 hwords; 22097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 len; 22197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 bth0; 22297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 bth2; 2234ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu); 22497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan char newreq; 2254ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell unsigned long flags; 2264ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell int ret = 0; 2274ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell 2284ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell ohdr = &qp->s_hdr.u.oth; 2294ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell if (qp->remote_ah_attr.ah_flags & IB_AH_GRH) 2304ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell ohdr = &qp->s_hdr.u.l.oth; 2314ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell 2324ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell /* 2334ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell * The lock is needed to synchronize between the sending tasklet, 2344ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell * the receive interrupt handler, and timeout resends. 2354ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell */ 2364ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell spin_lock_irqsave(&qp->s_lock, flags); 23797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 2383859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* Sending responses has higher priority over sending requests. */ 2393859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if ((qp->r_head_ack_queue != qp->s_tail_ack_queue || 2403859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell (qp->s_flags & IPATH_S_ACK_PENDING) || 241154257f3626ea6dd96781fac0896c3f27fe2b0a1Ralph Campbell qp->s_ack_state != OP(ACKNOWLEDGE)) && 2424ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell ipath_make_rc_ack(dev, qp, ohdr, pmtu)) 2433859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto done; 2443859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 245e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK)) { 246e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (!(ib_ipath_state_ops[qp->state] & IPATH_FLUSH_SEND)) 247e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto bail; 248e509be898d8937634437caa474b57ac12795e5bcRalph Campbell /* We are in the error state, flush the work request. */ 249e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (qp->s_last == qp->s_head) 250e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto bail; 251e509be898d8937634437caa474b57ac12795e5bcRalph Campbell /* If DMAs are in progress, we can't flush immediately. */ 252e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (atomic_read(&qp->s_dma_busy)) { 253e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->s_flags |= IPATH_S_WAIT_DMA; 254e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto bail; 255e509be898d8937634437caa474b57ac12795e5bcRalph Campbell } 256e509be898d8937634437caa474b57ac12795e5bcRalph Campbell wqe = get_swqe_ptr(qp, qp->s_last); 257e509be898d8937634437caa474b57ac12795e5bcRalph Campbell ipath_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); 258e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto done; 259e509be898d8937634437caa474b57ac12795e5bcRalph Campbell } 26097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 261e509be898d8937634437caa474b57ac12795e5bcRalph Campbell /* Leave BUSY set until RNR timeout. */ 262e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (qp->s_rnr_timeout) { 263e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->s_flags |= IPATH_S_WAITING; 2643859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 2656022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan } 2666022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan 26797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* header size in 32-bit words LRH+BTH = (8+12)/4. */ 26897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords = 5; 2691dd6a1be1416be48cafda9e63a614f26f0428d10Ralph Campbell bth0 = 1 << 22; /* Set M bit */ 27097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 27197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Send a request. */ 27297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe = get_swqe_ptr(qp, qp->s_cur); 27397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan switch (qp->s_state) { 27497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan default: 275e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (!(ib_ipath_state_ops[qp->state] & 276e509be898d8937634437caa474b57ac12795e5bcRalph Campbell IPATH_PROCESS_NEXT_SEND_OK)) 277e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto bail; 27897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 27997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Resend an old request or start a new one. 28097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 28197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * We keep track of the current SWQE so that 28297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * we don't reset the "furthest progress" state 28397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * if we need to back up. 28497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 28597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan newreq = 0; 28697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_cur == qp->s_tail) { 28797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check if send work queue is empty. */ 28897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_tail == qp->s_head) 2893859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 2903859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 2913859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * If a fence is requested, wait for previous 2923859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * RDMA read and atomic operations to finish. 2933859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 2943859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if ((wqe->wr.send_flags & IB_SEND_FENCE) && 2953859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_num_rd_atomic) { 2963859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_flags |= IPATH_S_FENCE_PENDING; 2973859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 2983859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 2996022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan wqe->psn = qp->s_next_psn; 30097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan newreq = 1; 30197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 30297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 30397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Note that we have to be careful not to modify the 30497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * original work request since we may need to resend 30597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * it. 30697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 30713b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell len = wqe->length; 30897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ss = &qp->s_sge; 30997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth2 = 0; 31097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan switch (wqe->wr.opcode) { 31197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case IB_WR_SEND: 31297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case IB_WR_SEND_WITH_IMM: 31397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* If no credit, return. */ 31497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_lsn != (u32) -1 && 315e509be898d8937634437caa474b57ac12795e5bcRalph Campbell ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) { 316e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->s_flags |= IPATH_S_WAIT_SSN_CREDIT; 3173859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 318e509be898d8937634437caa474b57ac12795e5bcRalph Campbell } 31997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe->lpsn = wqe->psn; 32097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (len > pmtu) { 32197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe->lpsn += (len - 1) / pmtu; 32297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(SEND_FIRST); 32397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = pmtu; 32497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 32597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 32697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_SEND) 32797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(SEND_ONLY); 32897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 32997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE); 33097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Immediate data comes after the BTH */ 3310f39cf3d54e67a705773fd0ec56ca3dcd3e9272fRoland Dreier ohdr->u.imm_data = wqe->wr.ex.imm_data; 33297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords += 1; 33397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 33497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.send_flags & IB_SEND_SOLICITED) 33597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth0 |= 1 << 23; 33697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth2 = 1 << 31; /* Request ACK. */ 33797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (++qp->s_cur == qp->s_size) 33897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur = 0; 33997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 34097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 34197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case IB_WR_RDMA_WRITE: 3426700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (newreq && qp->s_lsn != (u32) -1) 34397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_lsn++; 34497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 34597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case IB_WR_RDMA_WRITE_WITH_IMM: 34697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* If no credit, return. */ 34797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_lsn != (u32) -1 && 348e509be898d8937634437caa474b57ac12795e5bcRalph Campbell ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) { 349e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->s_flags |= IPATH_S_WAIT_SSN_CREDIT; 3503859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 351e509be898d8937634437caa474b57ac12795e5bcRalph Campbell } 35297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.rc.reth.vaddr = 35397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan cpu_to_be64(wqe->wr.wr.rdma.remote_addr); 35497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.rc.reth.rkey = 35597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan cpu_to_be32(wqe->wr.wr.rdma.rkey); 35697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.rc.reth.length = cpu_to_be32(len); 3573859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell hwords += sizeof(struct ib_reth) / sizeof(u32); 35897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe->lpsn = wqe->psn; 35997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (len > pmtu) { 36097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe->lpsn += (len - 1) / pmtu; 36197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(RDMA_WRITE_FIRST); 36297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = pmtu; 36397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 36497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 36597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_RDMA_WRITE) 36697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(RDMA_WRITE_ONLY); 36797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 36897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = 36997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE); 3706700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan /* Immediate data comes after RETH */ 3710f39cf3d54e67a705773fd0ec56ca3dcd3e9272fRoland Dreier ohdr->u.rc.imm_data = wqe->wr.ex.imm_data; 37297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords += 1; 37397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.send_flags & IB_SEND_SOLICITED) 37497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth0 |= 1 << 23; 37597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 37697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth2 = 1 << 31; /* Request ACK. */ 37797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (++qp->s_cur == qp->s_size) 37897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur = 0; 37997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 38097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 38197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case IB_WR_RDMA_READ: 3823859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 3833859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Don't allow more operations to be started 3843859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * than the QP limits allow. 3853859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 38697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (newreq) { 3873859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (qp->s_num_rd_atomic >= 3883859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_max_rd_atomic) { 3893859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_flags |= IPATH_S_RDMAR_PENDING; 3903859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 3913859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 3923859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_num_rd_atomic++; 3936700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (qp->s_lsn != (u32) -1) 3946700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_lsn++; 39597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 39697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Adjust s_next_psn to count the 39797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * expected number of responses. 39897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 39997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (len > pmtu) 40097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_next_psn += (len - 1) / pmtu; 40197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe->lpsn = qp->s_next_psn++; 40297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 4033859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.rc.reth.vaddr = 4043859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell cpu_to_be64(wqe->wr.wr.rdma.remote_addr); 4053859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.rc.reth.rkey = 4063859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell cpu_to_be32(wqe->wr.wr.rdma.rkey); 4073859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.rc.reth.length = cpu_to_be32(len); 4083859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_state = OP(RDMA_READ_REQUEST); 4093859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32); 41097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ss = NULL; 41197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = 0; 41297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (++qp->s_cur == qp->s_size) 41397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur = 0; 41497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 41597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 41697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case IB_WR_ATOMIC_CMP_AND_SWP: 41797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case IB_WR_ATOMIC_FETCH_AND_ADD: 4183859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 4193859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Don't allow more operations to be started 4203859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * than the QP limits allow. 4213859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 42297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (newreq) { 4233859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (qp->s_num_rd_atomic >= 4243859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_max_rd_atomic) { 4253859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_flags |= IPATH_S_RDMAR_PENDING; 4263859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 4273859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 4283859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_num_rd_atomic++; 4296700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (qp->s_lsn != (u32) -1) 4306700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_lsn++; 43197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe->lpsn = wqe->psn; 43297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 4333859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) { 4343859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_state = OP(COMPARE_SWAP); 4353859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.atomic_eth.swap_data = cpu_to_be64( 4363859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.wr.atomic.swap); 4373859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.atomic_eth.compare_data = cpu_to_be64( 4383859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.wr.atomic.compare_add); 4393859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } else { 4403859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_state = OP(FETCH_ADD); 4413859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.atomic_eth.swap_data = cpu_to_be64( 4423859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.wr.atomic.compare_add); 4433859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.atomic_eth.compare_data = 0; 4443859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 4453859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.atomic_eth.vaddr[0] = cpu_to_be32( 4463859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.wr.atomic.remote_addr >> 32); 4473859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.atomic_eth.vaddr[1] = cpu_to_be32( 4483859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.wr.atomic.remote_addr); 4493859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ohdr->u.atomic_eth.rkey = cpu_to_be32( 4503859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.wr.atomic.rkey); 4513859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell hwords += sizeof(struct ib_atomic_eth) / sizeof(u32); 45297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ss = NULL; 45397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = 0; 4543859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (++qp->s_cur == qp->s_size) 4553859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_cur = 0; 45697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 45797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 45897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan default: 4593859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto bail; 46097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 46113b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell qp->s_sge.sge = wqe->sg_list[0]; 46213b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell qp->s_sge.sg_list = wqe->sg_list + 1; 46313b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell qp->s_sge.num_sge = wqe->wr.num_sge; 46413b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell qp->s_len = wqe->length; 46597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (newreq) { 46697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_tail++; 46797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_tail >= qp->s_size) 46897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_tail = 0; 46997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 47013b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell bth2 |= qp->s_psn & IPATH_PSN_MASK; 47113b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell if (wqe->wr.opcode == IB_WR_RDMA_READ) 47213b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell qp->s_psn = wqe->lpsn + 1; 47313b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell else { 47413b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell qp->s_psn++; 4756f5c407460bba332d6bee52e19f2305539395511Ralph Campbell if (ipath_cmp24(qp->s_psn, qp->s_next_psn) > 0) 47613b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell qp->s_next_psn = qp->s_psn; 47713b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell } 47812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan /* 47912eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * Put the QP on the pending list so lost ACKs will cause 48012eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * a retry. More than one request can be pending so the 48112eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * QP may already be on the dev->pending list. 48212eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan */ 48397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_lock(&dev->pending_lock); 48494b8d9f98d7f535037eb9845b81396f667b4f727Bryan O'Sullivan if (list_empty(&qp->timerwait)) 48597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan list_add_tail(&qp->timerwait, 48697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan &dev->pending[dev->pending_index]); 48797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_unlock(&dev->pending_lock); 48897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 48997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 49097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_FIRST): 49197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 49212eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * This case can only happen if a send is restarted. 49312eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * See ipath_restart_rc(). 49497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 49597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_init_restart(qp, wqe); 49697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 49797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_FIRST): 49897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(SEND_MIDDLE); 49997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 50097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_MIDDLE): 50127b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan bth2 = qp->s_psn++ & IPATH_PSN_MASK; 5026f5c407460bba332d6bee52e19f2305539395511Ralph Campbell if (ipath_cmp24(qp->s_psn, qp->s_next_psn) > 0) 50397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_next_psn = qp->s_psn; 50497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ss = &qp->s_sge; 50597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = qp->s_len; 50697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (len > pmtu) { 50797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = pmtu; 50897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 50997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 51097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_SEND) 51197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(SEND_LAST); 51297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 51397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); 51497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Immediate data comes after the BTH */ 5150f39cf3d54e67a705773fd0ec56ca3dcd3e9272fRoland Dreier ohdr->u.imm_data = wqe->wr.ex.imm_data; 51697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords += 1; 51797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 51897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.send_flags & IB_SEND_SOLICITED) 51997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth0 |= 1 << 23; 52097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth2 |= 1 << 31; /* Request ACK. */ 52197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur++; 52297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_cur >= qp->s_size) 52397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur = 0; 52497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 52597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 52697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_LAST): 52797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 52897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * This case can only happen if a RDMA write is restarted. 52997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * See ipath_restart_rc(). 53097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 53197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_init_restart(qp, wqe); 53297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 53397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_FIRST): 53497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(RDMA_WRITE_MIDDLE); 53597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 53697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_MIDDLE): 53727b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan bth2 = qp->s_psn++ & IPATH_PSN_MASK; 5386f5c407460bba332d6bee52e19f2305539395511Ralph Campbell if (ipath_cmp24(qp->s_psn, qp->s_next_psn) > 0) 53997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_next_psn = qp->s_psn; 54097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ss = &qp->s_sge; 54197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = qp->s_len; 54297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (len > pmtu) { 54397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = pmtu; 54497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 54597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 54697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_RDMA_WRITE) 54797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(RDMA_WRITE_LAST); 54897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 54997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); 55097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Immediate data comes after the BTH */ 5510f39cf3d54e67a705773fd0ec56ca3dcd3e9272fRoland Dreier ohdr->u.imm_data = wqe->wr.ex.imm_data; 55297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hwords += 1; 55397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.send_flags & IB_SEND_SOLICITED) 55497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth0 |= 1 << 23; 55597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 55697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan bth2 |= 1 << 31; /* Request ACK. */ 55797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur++; 55897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_cur >= qp->s_size) 55997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur = 0; 56097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 56197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 56297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_MIDDLE): 56397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 56497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * This case can only happen if a RDMA read is restarted. 56597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * See ipath_restart_rc(). 56697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 56797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_init_restart(qp, wqe); 56827b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan len = ((qp->s_psn - wqe->psn) & IPATH_PSN_MASK) * pmtu; 56997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.rc.reth.vaddr = 57097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan cpu_to_be64(wqe->wr.wr.rdma.remote_addr + len); 57197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.rc.reth.rkey = 57297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan cpu_to_be32(wqe->wr.wr.rdma.rkey); 57397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->u.rc.reth.length = cpu_to_be32(qp->s_len); 57497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(RDMA_READ_REQUEST); 5753859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32); 576fab01fc56063dafcc083f481ac0f9e6b5a576dd6Ralph Campbell bth2 = qp->s_psn & IPATH_PSN_MASK; 577fab01fc56063dafcc083f481ac0f9e6b5a576dd6Ralph Campbell qp->s_psn = wqe->lpsn + 1; 57897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ss = NULL; 57997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan len = 0; 58097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur++; 58197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_cur == qp->s_size) 58297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur = 0; 58397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 58497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 5856022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan if (ipath_cmp24(qp->s_psn, qp->s_last_psn + IPATH_PSN_CREDIT - 1) >= 0) 5866022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan bth2 |= 1 << 31; /* Request ACK. */ 58797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_len -= len; 58897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_hdrwords = hwords; 58997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur_sge = ss; 59097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur_size = len; 5914ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell ipath_make_ruc_header(dev, qp, ohdr, bth0 | (qp->s_state << 24), bth2); 5923859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbelldone: 5934ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell ret = 1; 594e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto unlock; 595e509be898d8937634437caa474b57ac12795e5bcRalph Campbell 5963859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbellbail: 597e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->s_flags &= ~IPATH_S_BUSY; 598e509be898d8937634437caa474b57ac12795e5bcRalph Campbellunlock: 5994ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 6004ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell return ret; 60197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 60297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 60397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 604ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * send_rc_ack - Construct an ACK packet and send it 605ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * @qp: a pointer to the QP 60697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 607ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * This is called from ipath_rc_rcv() and only uses the receive 608ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * side QP state. 6093859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Note that RDMA reads and atomics are handled in the 6103859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * send side QP state and tasklet. 61197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 61297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanstatic void send_rc_ack(struct ipath_qp *qp) 61397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 61497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_ibdev *dev = to_idev(qp->ibqp.device); 615d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell struct ipath_devdata *dd; 61697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u16 lrh0; 61797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 bth0; 618ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan u32 hwords; 619d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell u32 __iomem *piobuf; 620ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan struct ipath_ib_header hdr; 62197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_other_headers *ohdr; 62235ff032e65ab5cc03bbba46cefece7376c7c562fRalph Campbell unsigned long flags; 62397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 624d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 625d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell 6263859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* Don't send ACK or NAK if a RDMA read or atomic is pending. */ 627154257f3626ea6dd96781fac0896c3f27fe2b0a1Ralph Campbell if (qp->r_head_ack_queue != qp->s_tail_ack_queue || 628154257f3626ea6dd96781fac0896c3f27fe2b0a1Ralph Campbell (qp->s_flags & IPATH_S_ACK_PENDING) || 629154257f3626ea6dd96781fac0896c3f27fe2b0a1Ralph Campbell qp->s_ack_state != OP(ACKNOWLEDGE)) 6303859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto queue_ack; 6313859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 632d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 633d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell 634e509be898d8937634437caa474b57ac12795e5bcRalph Campbell /* Don't try to send ACKs if the link isn't ACTIVE */ 635d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell dd = dev->dd; 636e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (!(dd->ipath_flags & IPATH_LINKACTIVE)) 637e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto done; 638e509be898d8937634437caa474b57ac12795e5bcRalph Campbell 639d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell piobuf = ipath_getpiobuf(dd, 0, NULL); 640d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell if (!piobuf) { 641d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell /* 642d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell * We are out of PIO buffers at the moment. 643d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell * Pass responsibility for sending the ACK to the 644d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell * send tasklet so that when a PIO buffer becomes 645d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell * available, the ACK is sent ahead of other outgoing 646d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell * packets. 647d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell */ 648d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 649d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell goto queue_ack; 650d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell } 651d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell 65297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Construct the header. */ 653ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan ohdr = &hdr.u.oth; 65427b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan lrh0 = IPATH_LRH_BTH; 65597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* header size in 32-bit words LRH+BTH+AETH = (8+12+4)/4. */ 656ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hwords = 6; 65797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) { 658ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hwords += ipath_make_grh(dev, &hdr.u.l.grh, 659ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan &qp->remote_ah_attr.grh, 660ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hwords, 0); 661ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan ohdr = &hdr.u.l.oth; 66227b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan lrh0 = IPATH_LRH_GRH; 66397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 66412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan /* read pkey_index w/o lock (its atomic) */ 665d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell bth0 = ipath_get_pkey(dd, qp->s_pkey_index) | 6661dd6a1be1416be48cafda9e63a614f26f0428d10Ralph Campbell (OP(ACKNOWLEDGE) << 24) | (1 << 22); 66712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan if (qp->r_nak_state) 66827b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan ohdr->u.aeth = cpu_to_be32((qp->r_msn & IPATH_MSN_MASK) | 66912eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan (qp->r_nak_state << 67027b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan IPATH_AETH_CREDIT_SHIFT)); 67112eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan else 67212eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan ohdr->u.aeth = ipath_compute_aeth(qp); 67397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan lrh0 |= qp->remote_ah_attr.sl << 4; 674ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hdr.lrh[0] = cpu_to_be16(lrh0); 675ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid); 676ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hdr.lrh[2] = cpu_to_be16(hwords + SIZE_OF_CRC); 677940358967599ba9057b3c51ba906e1cd5b984729Ralph Campbell hdr.lrh[3] = cpu_to_be16(dd->ipath_lid | 678940358967599ba9057b3c51ba906e1cd5b984729Ralph Campbell qp->remote_ah_attr.src_path_bits); 67997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->bth[0] = cpu_to_be32(bth0); 68097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr->bth[1] = cpu_to_be32(qp->remote_qpn); 68127b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan ohdr->bth[2] = cpu_to_be32(qp->r_ack_psn & IPATH_PSN_MASK); 68297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 683d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell writeq(hwords + 1, piobuf); 6843859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 685d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell if (dd->ipath_flags & IPATH_PIO_FLUSH_WC) { 686d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell u32 *hdrp = (u32 *) &hdr; 687d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell 688d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell ipath_flush_wc(); 689d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell __iowrite32_copy(piobuf + 2, hdrp, hwords - 1); 690d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell ipath_flush_wc(); 691d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell __raw_writel(hdrp[hwords - 1], piobuf + hwords + 1); 692d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell } else 693d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell __iowrite32_copy(piobuf + 2, (u32 *) &hdr, hwords); 694d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell 695d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell ipath_flush_wc(); 696d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell 697d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell dev->n_unicast_xmit++; 698d98b1937768c9f4e4420bd25406e5f0304d224bbRalph Campbell goto done; 6993859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 7003859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbellqueue_ack: 701e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK) { 702e509be898d8937634437caa474b57ac12795e5bcRalph Campbell dev->n_rc_qacks++; 703e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->s_flags |= IPATH_S_ACK_PENDING; 704e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->s_nak_state = qp->r_nak_state; 705e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->s_ack_psn = qp->r_ack_psn; 706e509be898d8937634437caa474b57ac12795e5bcRalph Campbell 707e509be898d8937634437caa474b57ac12795e5bcRalph Campbell /* Schedule the send tasklet. */ 708e509be898d8937634437caa474b57ac12795e5bcRalph Campbell ipath_schedule_send(qp); 709e509be898d8937634437caa474b57ac12795e5bcRalph Campbell } 71035ff032e65ab5cc03bbba46cefece7376c7c562fRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 7113859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbelldone: 7123859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell return; 71397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 71497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 71597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 7166700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * reset_psn - reset the QP state to send starting from PSN 7176700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * @qp: the QP 7186700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * @psn: the packet sequence number to restart at 7196700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * 7206700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * This is called from ipath_rc_rcv() to process an incoming RC ACK 7216700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * for the given QP. 7226700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * Called at interrupt level with the QP s_lock held. 7236700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan */ 7246700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivanstatic void reset_psn(struct ipath_qp *qp, u32 psn) 7256700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan{ 7266700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan u32 n = qp->s_last; 7276700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan struct ipath_swqe *wqe = get_swqe_ptr(qp, n); 7286700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan u32 opcode; 7296700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 7306700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_cur = n; 7316700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 7326700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan /* 7336700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * If we are starting the request from the beginning, 7346700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * let the normal send code handle initialization. 7356700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan */ 7366700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (ipath_cmp24(psn, wqe->psn) <= 0) { 7376700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_state = OP(SEND_LAST); 7386700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan goto done; 7396700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan } 7406700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 7416700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan /* Find the work request opcode corresponding to the given PSN. */ 7426700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan opcode = wqe->wr.opcode; 7436700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan for (;;) { 7446700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan int diff; 7456700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 7466700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (++n == qp->s_size) 7476700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan n = 0; 7486700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (n == qp->s_tail) 7496700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan break; 7506700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan wqe = get_swqe_ptr(qp, n); 7516700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan diff = ipath_cmp24(psn, wqe->psn); 7526700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (diff < 0) 7536700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan break; 7546700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_cur = n; 7556700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan /* 7566700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * If we are starting the request from the beginning, 7576700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * let the normal send code handle initialization. 7586700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan */ 7596700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan if (diff == 0) { 7606700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_state = OP(SEND_LAST); 7616700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan goto done; 7626700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan } 7636700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan opcode = wqe->wr.opcode; 7646700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan } 7656700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 7666700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan /* 7676700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * Set the state to restart in the middle of a request. 7686700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * Don't change the s_sge, s_cur_sge, or s_cur_size. 769e509be898d8937634437caa474b57ac12795e5bcRalph Campbell * See ipath_make_rc_req(). 7706700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan */ 7716700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan switch (opcode) { 7726700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan case IB_WR_SEND: 7736700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan case IB_WR_SEND_WITH_IMM: 7746700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_state = OP(RDMA_READ_RESPONSE_FIRST); 7756700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan break; 7766700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 7776700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan case IB_WR_RDMA_WRITE: 7786700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan case IB_WR_RDMA_WRITE_WITH_IMM: 7796700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_state = OP(RDMA_READ_RESPONSE_LAST); 7806700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan break; 7816700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 7826700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan case IB_WR_RDMA_READ: 7836700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE); 7846700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan break; 7856700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 7866700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan default: 7876700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan /* 7886700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * This case shouldn't happen since its only 7896700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * one PSN per req. 7906700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan */ 7916700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_state = OP(SEND_LAST); 7926700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan } 7936700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivandone: 7946700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan qp->s_psn = psn; 7956700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan} 7966700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan 7976700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan/** 79897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ipath_restart_rc - back up requester to resend the last un-ACKed request 79997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: the QP to restart 80097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @psn: packet sequence number for the request 80197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @wc: the work completion request 80297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 80312eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * The QP s_lock should be held and interrupts disabled. 80497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 80553dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbellvoid ipath_restart_rc(struct ipath_qp *qp, u32 psn) 80697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 80797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last); 80897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_ibdev *dev; 80997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 81097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_retry == 0) { 81153dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell ipath_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR); 81253dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell ipath_error_qp(qp, IB_WC_WR_FLUSH_ERR); 81397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto bail; 81497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 81597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_retry--; 81697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 81797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 81897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Remove the QP from the timeout queue. 81997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Note: it may already have been removed by ipath_ib_timer(). 82097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 82197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev = to_idev(qp->ibqp.device); 82297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_lock(&dev->pending_lock); 82394b8d9f98d7f535037eb9845b81396f667b4f727Bryan O'Sullivan if (!list_empty(&qp->timerwait)) 82494b8d9f98d7f535037eb9845b81396f667b4f727Bryan O'Sullivan list_del_init(&qp->timerwait); 82553dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell if (!list_empty(&qp->piowait)) 82653dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell list_del_init(&qp->piowait); 82797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_unlock(&dev->pending_lock); 82897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 82997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_RDMA_READ) 83097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_rc_resends++; 83197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else 8326f5c407460bba332d6bee52e19f2305539395511Ralph Campbell dev->n_rc_resends += (qp->s_psn - psn) & IPATH_PSN_MASK; 83397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 8346700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan reset_psn(qp, psn); 835e509be898d8937634437caa474b57ac12795e5bcRalph Campbell ipath_schedule_send(qp); 83697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 83797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanbail: 83897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan return; 83997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 84097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 8416022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivanstatic inline void update_last_psn(struct ipath_qp *qp, u32 psn) 8426022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan{ 843e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->s_last_psn = psn; 8446022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan} 8456022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan 84697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 84797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * do_rc_ack - process an incoming RC ACK 84897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: the QP the ACK came in on 84997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @psn: the packet sequence number of the ACK 85097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @opcode: the opcode of the request that resulted in the ACK 85197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 8526700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * This is called from ipath_rc_rcv_resp() to process an incoming RC ACK 85397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * for the given QP. 85412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * Called at interrupt level with the QP s_lock held and interrupts disabled. 85597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Returns 1 if OK, 0 if current operation should be aborted (NAK). 85697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 85706ee109002672ac875558ec699b53cf08a865bd3Ralph Campbellstatic int do_rc_ack(struct ipath_qp *qp, u32 aeth, u32 psn, int opcode, 85806ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell u64 val) 85997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 86097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_ibdev *dev = to_idev(qp->ibqp.device); 86197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ib_wc wc; 86253dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell enum ib_wc_status status; 86397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_swqe *wqe; 86497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int ret = 0; 86513b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell u32 ack_psn; 86606ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell int diff; 86797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 86897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 86997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Remove the QP from the timeout queue (or RNR timeout queue). 87097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * If ipath_ib_timer() has already removed it, 87197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * it's OK since we hold the QP s_lock and ipath_restart_rc() 87297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * just won't find anything to restart if we ACK everything. 87397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 87497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_lock(&dev->pending_lock); 87594b8d9f98d7f535037eb9845b81396f667b4f727Bryan O'Sullivan if (!list_empty(&qp->timerwait)) 87694b8d9f98d7f535037eb9845b81396f667b4f727Bryan O'Sullivan list_del_init(&qp->timerwait); 87797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_unlock(&dev->pending_lock); 87897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 87997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 88097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Note that NAKs implicitly ACK outstanding SEND and RDMA write 88197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * requests and implicitly NAK RDMA read and atomic requests issued 88297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * before the NAK'ed request. The MSN won't include the NAK'ed 88397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * request but will include an ACK'ed request(s). 88497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 88513b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell ack_psn = psn; 88613b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell if (aeth >> 29) 88713b18c86176cab34ef30ef0a5962fcb0305f7269Ralph Campbell ack_psn--; 88897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe = get_swqe_ptr(qp, qp->s_last); 88997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 89097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 89197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * The MSN might be for a later WQE than the PSN indicates so 89297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * only complete WQEs that the PSN finishes. 89397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 89406ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell while ((diff = ipath_cmp24(ack_psn, wqe->lpsn)) >= 0) { 89506ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell /* 89606ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell * RDMA_READ_RESPONSE_ONLY is a special case since 89706ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell * we want to generate completion events for everything 89806ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell * before the RDMA read, copy the data, then generate 89906ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell * the completion for the read. 90006ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell */ 90106ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell if (wqe->wr.opcode == IB_WR_RDMA_READ && 90206ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell opcode == OP(RDMA_READ_RESPONSE_ONLY) && 90306ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell diff == 0) { 90406ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell ret = 1; 90506ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell goto bail; 90606ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell } 90797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 90897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * If this request is a RDMA read or atomic, and the ACK is 90997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * for a later operation, this ACK NAKs the RDMA read or 91097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * atomic. In other words, only a RDMA_READ_LAST or ONLY 91197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * can ACK a RDMA read and likewise for atomic ops. Note 91297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * that the NAK case can only happen if relaxed ordering is 91397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * used and requests are sent after an RDMA read or atomic 91497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * is sent but before the response is received. 91597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 91697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if ((wqe->wr.opcode == IB_WR_RDMA_READ && 91706ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell (opcode != OP(RDMA_READ_RESPONSE_LAST) || diff != 0)) || 91897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 91997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) && 92006ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell (opcode != OP(ATOMIC_ACKNOWLEDGE) || diff != 0))) { 92197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 92297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * The last valid PSN seen is the previous 92397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * request's. 92497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 9256022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan update_last_psn(qp, wqe->psn - 1); 92697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Retry this request. */ 92753dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell ipath_restart_rc(qp, wqe->psn); 92897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 92997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * No need to process the ACK/NAK since we are 93097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * restarting an earlier request. 93197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 93297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto bail; 93397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 93406ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 93506ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) 93606ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell *(u64 *) wqe->sg_list[0].vaddr = val; 9373859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (qp->s_num_rd_atomic && 9383859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell (wqe->wr.opcode == IB_WR_RDMA_READ || 9393859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 9403859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) { 9413859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_num_rd_atomic--; 9423859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* Restart sending task if fence is complete */ 943e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (((qp->s_flags & IPATH_S_FENCE_PENDING) && 944e509be898d8937634437caa474b57ac12795e5bcRalph Campbell !qp->s_num_rd_atomic) || 945e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->s_flags & IPATH_S_RDMAR_PENDING) 946e509be898d8937634437caa474b57ac12795e5bcRalph Campbell ipath_schedule_send(qp); 9473859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 94897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Post a send completion queue entry if requested. */ 9493859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (!(qp->s_flags & IPATH_S_SIGNAL_REQ_WR) || 95097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan (wqe->wr.send_flags & IB_SEND_SIGNALED)) { 95153dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell memset(&wc, 0, sizeof wc); 95297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.wr_id = wqe->wr.wr_id; 95397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.status = IB_WC_SUCCESS; 95497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode]; 95597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.byte_len = wqe->length; 956062dbb69f32b9ccea701b30f8cc0049482e6211fMichael S. Tsirkin wc.qp = &qp->ibqp; 95797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.src_qp = qp->remote_qpn; 95897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.slid = qp->remote_ah_attr.dlid; 95997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.sl = qp->remote_ah_attr.sl; 96097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 0); 96197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 96297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_retry = qp->s_retry_cnt; 96397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 96497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * If we are completing a request which is in the process of 96597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * being resent, we can stop resending it since we know the 96697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * responder has already seen it. 96797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 96897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_last == qp->s_cur) { 96997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (++qp->s_cur >= qp->s_size) 97097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_cur = 0; 9716f5c407460bba332d6bee52e19f2305539395511Ralph Campbell qp->s_last = qp->s_cur; 9726f5c407460bba332d6bee52e19f2305539395511Ralph Campbell if (qp->s_last == qp->s_tail) 9736f5c407460bba332d6bee52e19f2305539395511Ralph Campbell break; 97497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wqe = get_swqe_ptr(qp, qp->s_cur); 97597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_state = OP(SEND_LAST); 97697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_psn = wqe->psn; 9776f5c407460bba332d6bee52e19f2305539395511Ralph Campbell } else { 9786f5c407460bba332d6bee52e19f2305539395511Ralph Campbell if (++qp->s_last >= qp->s_size) 9796f5c407460bba332d6bee52e19f2305539395511Ralph Campbell qp->s_last = 0; 980e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (qp->state == IB_QPS_SQD && qp->s_last == qp->s_cur) 981e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->s_draining = 0; 9826f5c407460bba332d6bee52e19f2305539395511Ralph Campbell if (qp->s_last == qp->s_tail) 9836f5c407460bba332d6bee52e19f2305539395511Ralph Campbell break; 9846f5c407460bba332d6bee52e19f2305539395511Ralph Campbell wqe = get_swqe_ptr(qp, qp->s_last); 98597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 98697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 98797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 98897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan switch (aeth >> 29) { 98997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case 0: /* ACK */ 99097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_rc_acks++; 99197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* If this is a partial ACK, reset the retransmit timer. */ 99297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_last != qp->s_tail) { 99397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_lock(&dev->pending_lock); 994f4ad1bcc4425a772ea584e1f24abadc64c2b839fRalph Campbell if (list_empty(&qp->timerwait)) 995f4ad1bcc4425a772ea584e1f24abadc64c2b839fRalph Campbell list_add_tail(&qp->timerwait, 996f4ad1bcc4425a772ea584e1f24abadc64c2b839fRalph Campbell &dev->pending[dev->pending_index]); 99797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_unlock(&dev->pending_lock); 9986f5c407460bba332d6bee52e19f2305539395511Ralph Campbell /* 9996f5c407460bba332d6bee52e19f2305539395511Ralph Campbell * If we get a partial ACK for a resent operation, 10006f5c407460bba332d6bee52e19f2305539395511Ralph Campbell * we can stop resending the earlier packets and 10016f5c407460bba332d6bee52e19f2305539395511Ralph Campbell * continue with the next packet the receiver wants. 10026f5c407460bba332d6bee52e19f2305539395511Ralph Campbell */ 10036f5c407460bba332d6bee52e19f2305539395511Ralph Campbell if (ipath_cmp24(qp->s_psn, psn) <= 0) { 10046f5c407460bba332d6bee52e19f2305539395511Ralph Campbell reset_psn(qp, psn + 1); 1005e509be898d8937634437caa474b57ac12795e5bcRalph Campbell ipath_schedule_send(qp); 10066f5c407460bba332d6bee52e19f2305539395511Ralph Campbell } 10076f5c407460bba332d6bee52e19f2305539395511Ralph Campbell } else if (ipath_cmp24(qp->s_psn, psn) <= 0) { 10086f5c407460bba332d6bee52e19f2305539395511Ralph Campbell qp->s_state = OP(SEND_LAST); 10096f5c407460bba332d6bee52e19f2305539395511Ralph Campbell qp->s_psn = psn + 1; 101097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 101197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_get_credit(qp, aeth); 101297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_rnr_retry = qp->s_rnr_retry_cnt; 101397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_retry = qp->s_retry_cnt; 10146022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan update_last_psn(qp, psn); 101597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ret = 1; 101697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto bail; 101797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 101897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case 1: /* RNR NAK */ 101997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_rnr_naks++; 10206f5c407460bba332d6bee52e19f2305539395511Ralph Campbell if (qp->s_last == qp->s_tail) 10216f5c407460bba332d6bee52e19f2305539395511Ralph Campbell goto bail; 102297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_rnr_retry == 0) { 102353dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell status = IB_WC_RNR_RETRY_EXC_ERR; 102497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto class_b; 102597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 102697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->s_rnr_retry_cnt < 7) 102797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_rnr_retry--; 102897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 10296700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan /* The last valid PSN is the previous PSN. */ 10306022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan update_last_psn(qp, psn - 1); 103197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 10326f5c407460bba332d6bee52e19f2305539395511Ralph Campbell if (wqe->wr.opcode == IB_WR_RDMA_READ) 10336f5c407460bba332d6bee52e19f2305539395511Ralph Campbell dev->n_rc_resends++; 10346f5c407460bba332d6bee52e19f2305539395511Ralph Campbell else 10356f5c407460bba332d6bee52e19f2305539395511Ralph Campbell dev->n_rc_resends += 10366f5c407460bba332d6bee52e19f2305539395511Ralph Campbell (qp->s_psn - psn) & IPATH_PSN_MASK; 103797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 10386700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan reset_psn(qp, psn); 103997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 104097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_rnr_timeout = 104127b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan ib_ipath_rnr_table[(aeth >> IPATH_AETH_CREDIT_SHIFT) & 104227b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan IPATH_AETH_CREDIT_MASK]; 104397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_insert_rnr_queue(qp); 1044e509be898d8937634437caa474b57ac12795e5bcRalph Campbell ipath_schedule_send(qp); 104597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto bail; 104697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 104797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case 3: /* NAK */ 10486f5c407460bba332d6bee52e19f2305539395511Ralph Campbell if (qp->s_last == qp->s_tail) 10496f5c407460bba332d6bee52e19f2305539395511Ralph Campbell goto bail; 10506f5c407460bba332d6bee52e19f2305539395511Ralph Campbell /* The last valid PSN is the previous PSN. */ 10516f5c407460bba332d6bee52e19f2305539395511Ralph Campbell update_last_psn(qp, psn - 1); 105227b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan switch ((aeth >> IPATH_AETH_CREDIT_SHIFT) & 105327b678dd04a636f2c351816f4b3042c8815d4e9dBryan O'Sullivan IPATH_AETH_CREDIT_MASK) { 105497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case 0: /* PSN sequence error */ 105597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_seq_naks++; 105697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 10576f5c407460bba332d6bee52e19f2305539395511Ralph Campbell * Back up to the responder's expected PSN. 105897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Note that we might get a NAK in the middle of an 105997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * RDMA READ response which terminates the RDMA 106097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * READ. 106197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 106253dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell ipath_restart_rc(qp, psn); 106397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 106497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 106597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case 1: /* Invalid Request */ 106653dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell status = IB_WC_REM_INV_REQ_ERR; 106797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_other_naks++; 106897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto class_b; 106997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 107097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case 2: /* Remote Access Error */ 107153dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell status = IB_WC_REM_ACCESS_ERR; 107297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_other_naks++; 107397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto class_b; 107497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 107597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case 3: /* Remote Operation Error */ 107653dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell status = IB_WC_REM_OP_ERR; 107797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_other_naks++; 107897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan class_b: 107953dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell ipath_send_complete(qp, wqe, status); 108053dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell ipath_error_qp(qp, IB_WC_WR_FLUSH_ERR); 108197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 108297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 108397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan default: 108497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Ignore other reserved NAK error codes */ 108597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto reserved; 108697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 108797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->s_rnr_retry = qp->s_rnr_retry_cnt; 108897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto bail; 108997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 109097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan default: /* 2: reserved */ 109197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan reserved: 109297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Ignore reserved NAK codes. */ 109397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto bail; 109497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 109597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 109697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanbail: 109797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan return ret; 109897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 109997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 110097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 110197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ipath_rc_rcv_resp - process an incoming RC response packet 110297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @dev: the device this packet came in on 110397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @ohdr: the other headers for this packet 110497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @data: the packet data 110597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @tlen: the packet length 110697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: the QP for this packet 110797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @opcode: the opcode for this packet 110897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @psn: the packet sequence number for this packet 110997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @hdrsize: the header length 111097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @pmtu: the path MTU 111197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @header_in_data: true if part of the header data is in the data buffer 111297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 111397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * This is called from ipath_rc_rcv() to process an incoming RC response 111497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * packet for the given QP. 111597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Called at interrupt level. 111697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 111797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanstatic inline void ipath_rc_rcv_resp(struct ipath_ibdev *dev, 111897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_other_headers *ohdr, 111997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan void *data, u32 tlen, 112097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_qp *qp, 112197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 opcode, 112297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 psn, u32 hdrsize, u32 pmtu, 112397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int header_in_data) 112497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 11253859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell struct ipath_swqe *wqe; 112653dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell enum ib_wc_status status; 112797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan unsigned long flags; 112897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int diff; 112997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 pad; 113097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 aeth; 113106ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell u64 val; 113297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 113397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan spin_lock_irqsave(&qp->s_lock, flags); 113497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 1135e509be898d8937634437caa474b57ac12795e5bcRalph Campbell /* Double check we can process this now that we hold the s_lock. */ 1136e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) 1137e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto ack_done; 1138e509be898d8937634437caa474b57ac12795e5bcRalph Campbell 113997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Ignore invalid responses. */ 114097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (ipath_cmp24(psn, qp->s_next_psn) >= 0) 114197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto ack_done; 114297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 114397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Ignore duplicate responses. */ 114497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan diff = ipath_cmp24(psn, qp->s_last_psn); 114597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(diff <= 0)) { 114697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Update credits for "ghost" ACKs */ 114797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (diff == 0 && opcode == OP(ACKNOWLEDGE)) { 114897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!header_in_data) 114997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan aeth = be32_to_cpu(ohdr->u.aeth); 115097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 115197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan aeth = be32_to_cpu(((__be32 *) data)[0]); 115297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(__be32); 115397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 115497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if ((aeth >> 29) == 0) 115597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_get_credit(qp, aeth); 115697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 115797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto ack_done; 115897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 115997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 11603859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(qp->s_last == qp->s_tail)) 11613859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto ack_done; 11623859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe = get_swqe_ptr(qp, qp->s_last); 116353dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell status = IB_WC_SUCCESS; 11643859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 116597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan switch (opcode) { 116697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(ACKNOWLEDGE): 116797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(ATOMIC_ACKNOWLEDGE): 116897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_FIRST): 116997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!header_in_data) 117097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan aeth = be32_to_cpu(ohdr->u.aeth); 117197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 117297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan aeth = be32_to_cpu(((__be32 *) data)[0]); 117397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(__be32); 117497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 11753859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (opcode == OP(ATOMIC_ACKNOWLEDGE)) { 11763859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (!header_in_data) { 11773859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell __be32 *p = ohdr->u.at.atomic_ack_eth; 11783859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 11793859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell val = ((u64) be32_to_cpu(p[0]) << 32) | 11803859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell be32_to_cpu(p[1]); 11813859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } else 11823859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell val = be64_to_cpu(((__be64 *) data)[0]); 118306ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell } else 118406ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell val = 0; 118506ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell if (!do_rc_ack(qp, aeth, psn, opcode, val) || 118697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode != OP(RDMA_READ_RESPONSE_FIRST)) 118797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto ack_done; 118897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hdrsize += 4; 118906ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell wqe = get_swqe_ptr(qp, qp->s_last); 11903859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 1191dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell goto ack_op_err; 119274116f580b7279543340dd716a2af642f5c1c2c7Ralph Campbell qp->r_flags &= ~IPATH_R_RDMAR_SEQ; 119397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 11943859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * If this is a response to a resent RDMA read, we 11953859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * have to be careful to copy the data to the right 11963859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * location. 119797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 11983859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, 11993859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe, psn, pmtu); 12003859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto read_middle; 120197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 120297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_MIDDLE): 120397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* no AETH, no ACK */ 120497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(ipath_cmp24(psn, qp->s_last_psn + 1))) { 120597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_rdma_seq++; 120674116f580b7279543340dd716a2af642f5c1c2c7Ralph Campbell if (qp->r_flags & IPATH_R_RDMAR_SEQ) 120774116f580b7279543340dd716a2af642f5c1c2c7Ralph Campbell goto ack_done; 120874116f580b7279543340dd716a2af642f5c1c2c7Ralph Campbell qp->r_flags |= IPATH_R_RDMAR_SEQ; 120953dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell ipath_restart_rc(qp, qp->s_last_psn + 1); 121097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto ack_done; 121197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 12123859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 1213dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell goto ack_op_err; 12143859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell read_middle: 12157bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan if (unlikely(tlen != (hdrsize + pmtu + 4))) 1216dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell goto ack_len_err; 12173859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(pmtu >= qp->s_rdma_read_len)) 1218dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell goto ack_len_err; 12193859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 12207bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan /* We got a response so update the timeout. */ 12217bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan spin_lock(&dev->pending_lock); 12227bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan if (qp->s_rnr_timeout == 0 && !list_empty(&qp->timerwait)) 12237bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan list_move_tail(&qp->timerwait, 12247bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan &dev->pending[dev->pending_index]); 12257bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan spin_unlock(&dev->pending_lock); 1226450bb3875f5f5ab3679823c941d6045d16967370Patrick Marchand Latifi 1227450bb3875f5f5ab3679823c941d6045d16967370Patrick Marchand Latifi if (opcode == OP(RDMA_READ_RESPONSE_MIDDLE)) 1228450bb3875f5f5ab3679823c941d6045d16967370Patrick Marchand Latifi qp->s_retry = qp->s_retry_cnt; 1229450bb3875f5f5ab3679823c941d6045d16967370Patrick Marchand Latifi 12307bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan /* 12316700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * Update the RDMA receive state but do the copy w/o 12326700efdfc06d2dc9ef77988a00182c2ede0f1be0Bryan O'Sullivan * holding the locks and blocking interrupts. 12337bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan */ 12343859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_rdma_read_len -= pmtu; 12356022943eb4cb3cb9e43f27f1faeaba38e162d966Bryan O'Sullivan update_last_psn(qp, psn); 12367bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan spin_unlock_irqrestore(&qp->s_lock, flags); 12373859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ipath_copy_sge(&qp->s_rdma_read_sge, data, pmtu); 12387bbb15ea8543e2e49476a27b507be3b02828a124Bryan O'Sullivan goto bail; 123997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 12403859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(RDMA_READ_RESPONSE_ONLY): 124106ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell if (!header_in_data) 124206ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell aeth = be32_to_cpu(ohdr->u.aeth); 124306ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell else 124406ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell aeth = be32_to_cpu(((__be32 *) data)[0]); 124506ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell if (!do_rc_ack(qp, aeth, psn, opcode, 0)) 12463859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto ack_done; 1247dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell /* Get the number of bytes the message was padded by. */ 1248dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 1249dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell /* 1250dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell * Check that the data size is >= 0 && <= pmtu. 1251dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell * Remember to account for the AETH header (4) and 1252dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell * ICRC (4). 1253dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell */ 1254dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell if (unlikely(tlen < (hdrsize + pad + 8))) 1255dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell goto ack_len_err; 12563859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 12573859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * If this is a response to a resent RDMA read, we 12583859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * have to be careful to copy the data to the right 12593859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * location. 12603859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 126106ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell wqe = get_swqe_ptr(qp, qp->s_last); 12623859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, 12633859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe, psn, pmtu); 12643859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto read_last; 12653859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 126697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_READ_RESPONSE_LAST): 126797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* ACKs READ req. */ 126897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(ipath_cmp24(psn, qp->s_last_psn + 1))) { 126997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan dev->n_rdma_seq++; 127074116f580b7279543340dd716a2af642f5c1c2c7Ralph Campbell if (qp->r_flags & IPATH_R_RDMAR_SEQ) 127174116f580b7279543340dd716a2af642f5c1c2c7Ralph Campbell goto ack_done; 127274116f580b7279543340dd716a2af642f5c1c2c7Ralph Campbell qp->r_flags |= IPATH_R_RDMAR_SEQ; 127353dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell ipath_restart_rc(qp, qp->s_last_psn + 1); 127497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto ack_done; 127597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 12763859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 1277dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell goto ack_op_err; 1278dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell /* Get the number of bytes the message was padded by. */ 127997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 128097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 128197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Check that the data size is >= 1 && <= pmtu. 128297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Remember to account for the AETH header (4) and 128397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ICRC (4). 128497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 1285dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell if (unlikely(tlen <= (hdrsize + pad + 8))) 1286dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell goto ack_len_err; 1287dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell read_last: 128897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan tlen -= hdrsize + pad + 8; 1289dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell if (unlikely(tlen != qp->s_rdma_read_len)) 1290dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell goto ack_len_err; 129197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!header_in_data) 129297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan aeth = be32_to_cpu(ohdr->u.aeth); 129397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 129497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan aeth = be32_to_cpu(((__be32 *) data)[0]); 129597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(__be32); 129697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 12973859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ipath_copy_sge(&qp->s_rdma_read_sge, data, tlen); 129806ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell (void) do_rc_ack(qp, aeth, psn, 129906ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell OP(RDMA_READ_RESPONSE_LAST), 0); 130097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto ack_done; 130197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 130297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 1303dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbellack_op_err: 130453dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell status = IB_WC_LOC_QP_OP_ERR; 1305dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell goto ack_err; 1306dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbell 1307dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbellack_len_err: 130853dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell status = IB_WC_LOC_LEN_ERR; 1309dd5190b6be0f3e27b6a4933a6a6d2d59957fc748Ralph Campbellack_err: 131053dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell ipath_send_complete(qp, wqe, status); 131153dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell ipath_error_qp(qp, IB_WC_WR_FLUSH_ERR); 131253dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbellack_done: 13136ed89b9574776d4178f1ad754d20e4f1e5a4b6c8Ralph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 131497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanbail: 131597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan return; 131697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 131797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 131897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 131997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ipath_rc_rcv_error - process an incoming duplicate or error RC packet 132097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @dev: the device this packet came in on 132197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @ohdr: the other headers for this packet 132297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @data: the packet data 132397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: the QP for this packet 132497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @opcode: the opcode for this packet 132597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @psn: the packet sequence number for this packet 132697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @diff: the difference between the PSN and the expected PSN 132797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @header_in_data: true if part of the header data is in the data buffer 132897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 132997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * This is called from ipath_rc_rcv() to process an unexpected 133097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * incoming RC packet for the given QP. 133197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Called at interrupt level. 133297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Return 1 if no more processing is needed; otherwise return 0 to 13333859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * schedule a response to be sent. 133497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 133597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanstatic inline int ipath_rc_rcv_error(struct ipath_ibdev *dev, 133697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_other_headers *ohdr, 133797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan void *data, 133897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_qp *qp, 133997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 opcode, 134097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 psn, 134197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int diff, 134297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int header_in_data) 134397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 13443859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell struct ipath_ack_entry *e; 13453859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u8 i, prev; 13463859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell int old_req; 134735ff032e65ab5cc03bbba46cefece7376c7c562fRalph Campbell unsigned long flags; 134897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 134997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (diff > 0) { 135097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 135197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Packet sequence error. 135297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * A NAK will ACK earlier sends and RDMA writes. 13533859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Don't queue the NAK if we already sent one. 135497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 13553859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (!qp->r_nak_state) { 135612eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_nak_state = IB_NAK_PSN_ERROR; 135712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan /* Use the expected PSN. */ 135812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_ack_psn = qp->r_psn; 13593859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto send_ack; 136097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 13613859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto done; 136297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 136397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 136497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 136597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Handle a duplicate request. Don't re-execute SEND, RDMA 136697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * write or atomic op. Don't NAK errors, just silently drop 136797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * the duplicate request. Note that r_sge, r_len, and 136897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * r_rcv_len may be in use so don't modify them. 136997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 137097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * We are supposed to ACK the earliest duplicate PSN but we 137197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * can coalesce an outstanding duplicate ACK. We have to 137297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * send the earliest so that RDMA reads can be restarted at 137397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * the requester's expected PSN. 13743859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * 13753859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * First, find where this duplicate PSN falls within the 13763859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * ACKs previously sent. 137797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 13783859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell psn &= IPATH_PSN_MASK; 13793859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e = NULL; 13803859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell old_req = 1; 1381e509be898d8937634437caa474b57ac12795e5bcRalph Campbell 138235ff032e65ab5cc03bbba46cefece7376c7c562fRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 1383e509be898d8937634437caa474b57ac12795e5bcRalph Campbell /* Double check we can process this now that we hold the s_lock. */ 1384e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) 1385e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto unlock_done; 1386e509be898d8937634437caa474b57ac12795e5bcRalph Campbell 13873859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell for (i = qp->r_head_ack_queue; ; i = prev) { 13883859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (i == qp->s_tail_ack_queue) 13893859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell old_req = 0; 13903859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (i) 13913859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell prev = i - 1; 13923859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell else 13933859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell prev = IPATH_MAX_RDMA_ATOMIC; 13943859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (prev == qp->r_head_ack_queue) { 13953859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e = NULL; 13963859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell break; 13973859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 13983859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e = &qp->s_ack_queue[prev]; 13993859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (!e->opcode) { 14003859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e = NULL; 14013859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell break; 14023859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 140306ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell if (ipath_cmp24(psn, e->psn) >= 0) { 140406ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell if (prev == qp->s_tail_ack_queue) 140506ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell old_req = 0; 14063859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell break; 140706ee109002672ac875558ec699b53cf08a865bd3Ralph Campbell } 14083859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 14093859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell switch (opcode) { 14103859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(RDMA_READ_REQUEST): { 14113859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell struct ib_reth *reth; 14123859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u32 offset; 14133859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u32 len; 14143859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 14153859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 14163859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * If we didn't find the RDMA read request in the ack queue, 14173859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * or the send tasklet is already backed up to send an 14183859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * earlier entry, we can ignore this request. 14193859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 14203859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (!e || e->opcode != OP(RDMA_READ_REQUEST) || old_req) 14213859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto unlock_done; 142297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* RETH comes after BTH */ 142397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!header_in_data) 142497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan reth = &ohdr->u.rc.reth; 142597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 142697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan reth = (struct ib_reth *)data; 142797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(*reth); 142897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 142912eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan /* 14303859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Address range must be a subset of the original 14313859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * request and start on pmtu boundaries. 14323859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * We reuse the old ack_queue slot since the requester 14333859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * should not back up and request an earlier PSN for the 14343859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * same request. 143512eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan */ 14363859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell offset = ((psn - e->psn) & IPATH_PSN_MASK) * 14373859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ib_mtu_enum_to_int(qp->path_mtu); 14383859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = be32_to_cpu(reth->length); 14393859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(offset + len > e->rdma_sge.sge.sge_length)) 14403859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto unlock_done; 14413859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (len != 0) { 144297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 rkey = be32_to_cpu(reth->rkey); 144397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u64 vaddr = be64_to_cpu(reth->vaddr); 144497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int ok; 144597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 14463859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ok = ipath_rkey_ok(qp, &e->rdma_sge, 14473859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len, vaddr, rkey, 144897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan IB_ACCESS_REMOTE_READ); 14493859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(!ok)) 14503859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto unlock_done; 145197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } else { 14523859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sg_list = NULL; 14533859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.num_sge = 0; 14543859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.mr = NULL; 14553859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.vaddr = NULL; 14563859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.length = 0; 14573859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.sge_length = 0; 145897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 14593859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->psn = psn; 14603859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(ACKNOWLEDGE); 14613859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_tail_ack_queue = prev; 14623859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell break; 146312eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan } 146412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan 146597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(COMPARE_SWAP): 14663859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(FETCH_ADD): { 146797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 14683859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * If we didn't find the atomic request in the ack queue 14693859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * or the send tasklet is already backed up to send an 14703859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * earlier entry, we can ignore this request. 147197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 14723859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (!e || e->opcode != (u8) opcode || old_req) 14733859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto unlock_done; 14743859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(ACKNOWLEDGE); 14753859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_tail_ack_queue = prev; 14763859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell break; 14773859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 14783859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 14793859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell default: 14803859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (old_req) 14813859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto unlock_done; 14823859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 14833859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Resend the most recent ACK if this request is 14843859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * after all the previous RDMA reads and atomics. 14853859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 14863859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (i == qp->r_head_ack_queue) { 148735ff032e65ab5cc03bbba46cefece7376c7c562fRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 14883859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_nak_state = 0; 14893859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_ack_psn = qp->r_psn - 1; 14903859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto send_ack; 14913859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 14923859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* 14934ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell * Try to send a simple ACK to work around a Mellanox bug 14944ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell * which doesn't accept a RDMA read response or atomic 14954ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell * response as an ACK for earlier SENDs or RDMA writes. 14964ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell */ 14974ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell if (qp->r_head_ack_queue == qp->s_tail_ack_queue && 14984ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell !(qp->s_flags & IPATH_S_ACK_PENDING) && 14994ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp->s_ack_state == OP(ACKNOWLEDGE)) { 15004ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 15014ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp->r_nak_state = 0; 15024ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp->r_ack_psn = qp->s_ack_queue[i].psn - 1; 15034ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell goto send_ack; 15044ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell } 15054ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell /* 15063859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * Resend the RDMA read or atomic op which 15073859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell * ACKs this duplicate request. 15083859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell */ 15093859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_ack_state = OP(ACKNOWLEDGE); 15103859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_tail_ack_queue = i; 151197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 151297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 151312eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_nak_state = 0; 1514e509be898d8937634437caa474b57ac12795e5bcRalph Campbell ipath_schedule_send(qp); 151597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 15163859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbellunlock_done: 151735ff032e65ab5cc03bbba46cefece7376c7c562fRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 151897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivandone: 151997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan return 1; 15203859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 15213859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbellsend_ack: 15223859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell return 0; 152397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 152497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 152553dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbellvoid ipath_rc_error(struct ipath_qp *qp, enum ib_wc_status err) 15268d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivan{ 152735ff032e65ab5cc03bbba46cefece7376c7c562fRalph Campbell unsigned long flags; 1528d42b01b584b6f55f70c56f6a3dabc26f4982d30dRalph Campbell int lastwqe; 152935ff032e65ab5cc03bbba46cefece7376c7c562fRalph Campbell 153035ff032e65ab5cc03bbba46cefece7376c7c562fRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 1531d42b01b584b6f55f70c56f6a3dabc26f4982d30dRalph Campbell lastwqe = ipath_error_qp(qp, err); 153235ff032e65ab5cc03bbba46cefece7376c7c562fRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 1533d42b01b584b6f55f70c56f6a3dabc26f4982d30dRalph Campbell 1534d42b01b584b6f55f70c56f6a3dabc26f4982d30dRalph Campbell if (lastwqe) { 1535d42b01b584b6f55f70c56f6a3dabc26f4982d30dRalph Campbell struct ib_event ev; 1536d42b01b584b6f55f70c56f6a3dabc26f4982d30dRalph Campbell 1537d42b01b584b6f55f70c56f6a3dabc26f4982d30dRalph Campbell ev.device = qp->ibqp.device; 1538d42b01b584b6f55f70c56f6a3dabc26f4982d30dRalph Campbell ev.element.qp = &qp->ibqp; 1539d42b01b584b6f55f70c56f6a3dabc26f4982d30dRalph Campbell ev.event = IB_EVENT_QP_LAST_WQE_REACHED; 1540d42b01b584b6f55f70c56f6a3dabc26f4982d30dRalph Campbell qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); 1541d42b01b584b6f55f70c56f6a3dabc26f4982d30dRalph Campbell } 15428d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivan} 15438d0208cb59a43bf867e16b977c34c4d6cd618f59Bryan O'Sullivan 1544d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbellstatic inline void ipath_update_ack_queue(struct ipath_qp *qp, unsigned n) 1545d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell{ 1546d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell unsigned next; 1547d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell 1548d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell next = n + 1; 1549d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell if (next > IPATH_MAX_RDMA_ATOMIC) 1550d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell next = 0; 1551d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell if (n == qp->s_tail_ack_queue) { 1552d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell qp->s_tail_ack_queue = next; 1553d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell qp->s_ack_state = OP(ACKNOWLEDGE); 1554d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell } 1555d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell} 1556d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell 155797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan/** 155897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * ipath_rc_rcv - process an incoming RC packet 155997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @dev: the device this packet came in on 156097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @hdr: the header of this packet 156197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @has_grh: true if the header has a GRH 156297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @data: the packet data 156397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @tlen: the packet length 156497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * @qp: the QP for this packet 156597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * 156697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * This is called from ipath_qp_rcv() to process an incoming RC packet 156797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * for the given QP. 156897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Called at interrupt level. 156997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 157097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivanvoid ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 157197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int has_grh, void *data, u32 tlen, struct ipath_qp *qp) 157297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan{ 157397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ipath_other_headers *ohdr; 157497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 opcode; 157597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 hdrsize; 157697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 psn; 157797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 pad; 157897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ib_wc wc; 157997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu); 158097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int diff; 158197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ib_reth *reth; 158297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int header_in_data; 1583e509be898d8937634437caa474b57ac12795e5bcRalph Campbell unsigned long flags; 158497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 158510aeb0e6d8823c1cccf9edc8401c848745c128beBryan O'Sullivan /* Validate the SLID. See Ch. 9.6.1.5 */ 158610aeb0e6d8823c1cccf9edc8401c848745c128beBryan O'Sullivan if (unlikely(be16_to_cpu(hdr->lrh[3]) != qp->remote_ah_attr.dlid)) 158710aeb0e6d8823c1cccf9edc8401c848745c128beBryan O'Sullivan goto done; 158810aeb0e6d8823c1cccf9edc8401c848745c128beBryan O'Sullivan 158997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check for GRH */ 159097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!has_grh) { 159197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr = &hdr->u.oth; 159297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hdrsize = 8 + 12; /* LRH + BTH */ 159397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan psn = be32_to_cpu(ohdr->bth[2]); 159497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan header_in_data = 0; 159597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } else { 159697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ohdr = &hdr->u.l.oth; 159797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hdrsize = 8 + 40 + 12; /* LRH + GRH + BTH */ 159897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 159997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * The header with GRH is 60 bytes and the core driver sets 160097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * the eager header buffer size to 56 bytes so the last 4 160197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * bytes of the BTH header (PSN) is in the data buffer. 160297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 160334b2aafea38efdf02cd8107a6e1057e2a297c447Bryan O'Sullivan header_in_data = dev->dd->ipath_rcvhdrentsize == 16; 160497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (header_in_data) { 160597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan psn = be32_to_cpu(((__be32 *) data)[0]); 160697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(__be32); 160797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } else 160897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan psn = be32_to_cpu(ohdr->bth[2]); 160997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 161097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 161197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 161297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Process responses (ACKs) before anything else. Note that the 161397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * packet sequence number will be for something in the send work 161497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * queue rather than the expected receive packet sequence number. 161597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * In other words, this QP is the requester. 161697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 1617ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan opcode = be32_to_cpu(ohdr->bth[0]) >> 24; 161897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) && 161997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode <= OP(ATOMIC_ACKNOWLEDGE)) { 162097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_rc_rcv_resp(dev, ohdr, data, tlen, qp, opcode, psn, 162197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hdrsize, pmtu, header_in_data); 162212eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto done; 162397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 162497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 162597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Compute 24 bits worth of difference. */ 162697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan diff = ipath_cmp24(psn, qp->r_psn); 162797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(diff)) { 162897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (ipath_rc_rcv_error(dev, ohdr, data, qp, opcode, 162997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan psn, diff, header_in_data)) 163097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto done; 163112eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto send_ack; 163297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 163397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 163497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check for opcode sequence errors. */ 163597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan switch (qp->r_state) { 163697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_FIRST): 163797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_MIDDLE): 163897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (opcode == OP(SEND_MIDDLE) || 163997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(SEND_LAST) || 164097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(SEND_LAST_WITH_IMMEDIATE)) 164197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 164253dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell goto nack_inv; 164397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 164497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_FIRST): 164597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_MIDDLE): 164697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (opcode == OP(RDMA_WRITE_MIDDLE) || 164797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(RDMA_WRITE_LAST) || 164897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) 164997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 165097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_inv; 165197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 165297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan default: 165397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (opcode == OP(SEND_MIDDLE) || 165497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(SEND_LAST) || 165597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(SEND_LAST_WITH_IMMEDIATE) || 165697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(RDMA_WRITE_MIDDLE) || 165797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(RDMA_WRITE_LAST) || 165897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) 165997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_inv; 166012eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan /* 166112eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * Note that it is up to the requester to not send a new 166212eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * RDMA read or atomic operation before receiving an ACK 166312eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * for the previous operation. 166412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan */ 166597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 166697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 166797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 166853dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell memset(&wc, 0, sizeof wc); 166997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 167097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* OK, process the packet. */ 167197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan switch (opcode) { 167297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_FIRST): 167353dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell if (!ipath_get_rwqe(qp, 0)) 167453dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell goto rnr_nak; 167597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_rcv_len = 0; 167697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 167797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_MIDDLE): 167897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_MIDDLE): 167997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan send_middle: 168097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check for invalid length PMTU or posted rwqe len. */ 168197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(tlen != (hdrsize + pmtu + 4))) 168297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_inv; 168397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_rcv_len += pmtu; 168497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(qp->r_rcv_len > qp->r_len)) 168597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_inv; 168697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_copy_sge(&qp->r_sge, data, pmtu); 168797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 168897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 168997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE): 169097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* consume RWQE */ 169197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!ipath_get_rwqe(qp, 1)) 169297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto rnr_nak; 169397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto send_last_imm; 169497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 169597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_ONLY): 169697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_ONLY_WITH_IMMEDIATE): 169797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!ipath_get_rwqe(qp, 0)) 169897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto rnr_nak; 169997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_rcv_len = 0; 170097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (opcode == OP(SEND_ONLY)) 170197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto send_last; 170297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 170397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_LAST_WITH_IMMEDIATE): 170497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan send_last_imm: 170597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (header_in_data) { 170600f7ec36c9324928e4cd23f02e6d8550f30c32caSteve Wise wc.ex.imm_data = *(__be32 *) data; 170797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(__be32); 170897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } else { 170997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Immediate data comes after BTH */ 171000f7ec36c9324928e4cd23f02e6d8550f30c32caSteve Wise wc.ex.imm_data = ohdr->u.imm_data; 171197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 171297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hdrsize += 4; 171397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.wc_flags = IB_WC_WITH_IMM; 171497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* FALLTHROUGH */ 171597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(SEND_LAST): 171697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_LAST): 171797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan send_last: 171897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Get the number of bytes the message was padded by. */ 171997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 172097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check for invalid length. */ 172197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* XXX LAST len should be >= 1 */ 172297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(tlen < (hdrsize + pad + 4))) 172397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_inv; 172497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Don't count the CRC. */ 172597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan tlen -= (hdrsize + pad + 4); 172697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.byte_len = tlen + qp->r_rcv_len; 172797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(wc.byte_len > qp->r_len)) 172897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto nack_inv; 172997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_copy_sge(&qp->r_sge, data, tlen); 173012eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_msn++; 1731e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (!test_and_clear_bit(IPATH_R_WRID_VALID, &qp->r_aflags)) 173297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 173397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.wr_id = qp->r_wr_id; 173497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.status = IB_WC_SUCCESS; 17352bfc8e9edf200aeeca18ee44bcbf6bce65438a42Ralph Campbell if (opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE) || 17362bfc8e9edf200aeeca18ee44bcbf6bce65438a42Ralph Campbell opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE)) 17372bfc8e9edf200aeeca18ee44bcbf6bce65438a42Ralph Campbell wc.opcode = IB_WC_RECV_RDMA_WITH_IMM; 17382bfc8e9edf200aeeca18ee44bcbf6bce65438a42Ralph Campbell else 17392bfc8e9edf200aeeca18ee44bcbf6bce65438a42Ralph Campbell wc.opcode = IB_WC_RECV; 1740062dbb69f32b9ccea701b30f8cc0049482e6211fMichael S. Tsirkin wc.qp = &qp->ibqp; 174197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.src_qp = qp->remote_qpn; 174297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.slid = qp->remote_ah_attr.dlid; 174397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan wc.sl = qp->remote_ah_attr.sl; 174497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Signal completion event if the solicited bit is set. */ 174597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 174697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan (ohdr->bth[0] & 17479c3da0991754d480328eeaa2b90cb231a1cea9b6Harvey Harrison cpu_to_be32(1 << 23)) != 0); 174897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan break; 174997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 175097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_FIRST): 175197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_ONLY): 175297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): 1753b506e1dc59726a1c608f26e7294b9fe186255139Robert Walsh if (unlikely(!(qp->qp_access_flags & 1754b506e1dc59726a1c608f26e7294b9fe186255139Robert Walsh IB_ACCESS_REMOTE_WRITE))) 1755b506e1dc59726a1c608f26e7294b9fe186255139Robert Walsh goto nack_inv; 175697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* consume RWQE */ 175797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* RETH comes after BTH */ 175897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!header_in_data) 175997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan reth = &ohdr->u.rc.reth; 176097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 176197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan reth = (struct ib_reth *)data; 176297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(*reth); 176397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 176497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan hdrsize += sizeof(*reth); 176597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_len = be32_to_cpu(reth->length); 176697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_rcv_len = 0; 176797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (qp->r_len != 0) { 176897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 rkey = be32_to_cpu(reth->rkey); 176997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u64 vaddr = be64_to_cpu(reth->vaddr); 177097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int ok; 177197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 177297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check rkey & NAK */ 17736a553af286653818bb5831f1b351eefdc8a93b61Bryan O'Sullivan ok = ipath_rkey_ok(qp, &qp->r_sge, 177497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_len, vaddr, rkey, 177597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan IB_ACCESS_REMOTE_WRITE); 177612eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan if (unlikely(!ok)) 177712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto nack_acc; 177897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } else { 177997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_sge.sg_list = NULL; 178097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_sge.sge.mr = NULL; 178197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_sge.sge.vaddr = NULL; 178297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_sge.sge.length = 0; 178397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_sge.sge.sge_length = 0; 178497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 178597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (opcode == OP(RDMA_WRITE_FIRST)) 178697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto send_middle; 178797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else if (opcode == OP(RDMA_WRITE_ONLY)) 178897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto send_last; 178997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!ipath_get_rwqe(qp, 1)) 179097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto rnr_nak; 179197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan goto send_last_imm; 179297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 17933859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell case OP(RDMA_READ_REQUEST): { 17943859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell struct ipath_ack_entry *e; 17953859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u32 len; 17963859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u8 next; 17973859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 1798b506e1dc59726a1c608f26e7294b9fe186255139Robert Walsh if (unlikely(!(qp->qp_access_flags & 1799b506e1dc59726a1c608f26e7294b9fe186255139Robert Walsh IB_ACCESS_REMOTE_READ))) 1800b506e1dc59726a1c608f26e7294b9fe186255139Robert Walsh goto nack_inv; 18013859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell next = qp->r_head_ack_queue + 1; 18023859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (next > IPATH_MAX_RDMA_ATOMIC) 18033859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell next = 0; 1804e509be898d8937634437caa474b57ac12795e5bcRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 1805e509be898d8937634437caa474b57ac12795e5bcRalph Campbell /* Double check we can process this while holding the s_lock. */ 1806e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) 1807e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto unlock; 1808d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell if (unlikely(next == qp->s_tail_ack_queue)) { 1809d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell if (!qp->s_ack_queue[next].sent) 1810e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto nack_inv_unlck; 1811d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell ipath_update_ack_queue(qp, next); 1812d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell } 18133859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e = &qp->s_ack_queue[qp->r_head_ack_queue]; 181497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* RETH comes after BTH */ 181597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!header_in_data) 181697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan reth = &ohdr->u.rc.reth; 181797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan else { 181897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan reth = (struct ib_reth *)data; 181997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan data += sizeof(*reth); 182097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 18213859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell len = be32_to_cpu(reth->length); 18223859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (len) { 182397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 rkey = be32_to_cpu(reth->rkey); 182497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u64 vaddr = be64_to_cpu(reth->vaddr); 182597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan int ok; 182697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 182797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check rkey & NAK */ 18283859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell ok = ipath_rkey_ok(qp, &e->rdma_sge, len, vaddr, 18293859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell rkey, IB_ACCESS_REMOTE_READ); 18303859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(!ok)) 1831e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto nack_acc_unlck; 183297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 183397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * Update the next expected PSN. We add 1 later 183497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * below, so only add the remainder here. 183597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 18363859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (len > pmtu) 18373859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_psn += (len - 1) / pmtu; 183897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } else { 18393859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sg_list = NULL; 18403859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.num_sge = 0; 18413859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.mr = NULL; 18423859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.vaddr = NULL; 18433859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.length = 0; 18443859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->rdma_sge.sge.sge_length = 0; 184597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 18463859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->opcode = opcode; 1847d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell e->sent = 0; 18483859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->psn = psn; 184997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* 185097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * We need to increment the MSN here instead of when we 185197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * finish sending the result since a duplicate request would 185297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan * increment it more than once. 185397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan */ 185412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_msn++; 185597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_psn++; 185697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_state = opcode; 185712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_nak_state = 0; 18583859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_head_ack_queue = next; 185912eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan 1860e509be898d8937634437caa474b57ac12795e5bcRalph Campbell /* Schedule the send tasklet. */ 1861e509be898d8937634437caa474b57ac12795e5bcRalph Campbell ipath_schedule_send(qp); 186212eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan 1863e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto unlock; 18643859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell } 186597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 186697f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(COMPARE_SWAP): 186797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan case OP(FETCH_ADD): { 186897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan struct ib_atomic_eth *ateth; 18693859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell struct ipath_ack_entry *e; 187097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u64 vaddr; 18713859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell atomic64_t *maddr; 187297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u64 sdata; 187397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan u32 rkey; 18743859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell u8 next; 187597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 18763859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (unlikely(!(qp->qp_access_flags & 18773859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell IB_ACCESS_REMOTE_ATOMIC))) 1878b506e1dc59726a1c608f26e7294b9fe186255139Robert Walsh goto nack_inv; 18793859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell next = qp->r_head_ack_queue + 1; 18803859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (next > IPATH_MAX_RDMA_ATOMIC) 18813859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell next = 0; 1882e509be898d8937634437caa474b57ac12795e5bcRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 1883e509be898d8937634437caa474b57ac12795e5bcRalph Campbell /* Double check we can process this while holding the s_lock. */ 1884e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) 1885e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto unlock; 1886d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell if (unlikely(next == qp->s_tail_ack_queue)) { 1887d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell if (!qp->s_ack_queue[next].sent) 1888e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto nack_inv_unlck; 1889d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell ipath_update_ack_queue(qp, next); 1890d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell } 189197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (!header_in_data) 189297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ateth = &ohdr->u.atomic_eth; 18933859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell else 189497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan ateth = (struct ib_atomic_eth *)data; 18953859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell vaddr = ((u64) be32_to_cpu(ateth->vaddr[0]) << 32) | 18963859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell be32_to_cpu(ateth->vaddr[1]); 189797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan if (unlikely(vaddr & (sizeof(u64) - 1))) 1898e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto nack_inv_unlck; 189997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan rkey = be32_to_cpu(ateth->rkey); 190097f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Check rkey & NAK */ 19016a553af286653818bb5831f1b351eefdc8a93b61Bryan O'Sullivan if (unlikely(!ipath_rkey_ok(qp, &qp->r_sge, 190297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan sizeof(u64), vaddr, rkey, 190397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan IB_ACCESS_REMOTE_ATOMIC))) 1904e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto nack_acc_unlck; 190597f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Perform atomic OP and save result. */ 19063859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell maddr = (atomic64_t *) qp->r_sge.sge.vaddr; 190797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan sdata = be64_to_cpu(ateth->swap_data); 19083859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e = &qp->s_ack_queue[qp->r_head_ack_queue]; 19093859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->atomic_data = (opcode == OP(FETCH_ADD)) ? 19103859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell (u64) atomic64_add_return(sdata, maddr) - sdata : 19113859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr, 19123859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell be64_to_cpu(ateth->compare_data), 19133859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell sdata); 19143859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->opcode = opcode; 1915d781b129f1e8b3e2f369d8035a61a5233832e65cRalph Campbell e->sent = 0; 19163859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell e->psn = psn & IPATH_PSN_MASK; 191712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_msn++; 19183859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_psn++; 19193859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_state = opcode; 19203859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_nak_state = 0; 19213859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_head_ack_queue = next; 19223859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 1923e509be898d8937634437caa474b57ac12795e5bcRalph Campbell /* Schedule the send tasklet. */ 1924e509be898d8937634437caa474b57ac12795e5bcRalph Campbell ipath_schedule_send(qp); 19253859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell 1926e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto unlock; 192797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 192897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 192997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan default: 19303859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell /* NAK unknown opcodes. */ 19313859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell goto nack_inv; 193297f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan } 193397f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_psn++; 193497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan qp->r_state = opcode; 19353859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_ack_psn = psn; 193612eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan qp->r_nak_state = 0; 193797f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan /* Send an ACK if requested or required. */ 19383859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell if (psn & (1 << 31)) 193912eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto send_ack; 194012eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto done; 194197f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 194253dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbellrnr_nak: 194353dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell qp->r_nak_state = IB_RNR_NAK | qp->r_min_rnr_timer; 194453dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell qp->r_ack_psn = qp->r_psn; 194553dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell goto send_ack; 194653dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell 1947e509be898d8937634437caa474b57ac12795e5bcRalph Campbellnack_inv_unlck: 1948e509be898d8937634437caa474b57ac12795e5bcRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 194953dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbellnack_inv: 195053dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell ipath_rc_error(qp, IB_WC_LOC_QP_OP_ERR); 195153dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell qp->r_nak_state = IB_NAK_INVALID_REQUEST; 195253dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell qp->r_ack_psn = qp->r_psn; 195353dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell goto send_ack; 195453dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell 1955e509be898d8937634437caa474b57ac12795e5bcRalph Campbellnack_acc_unlck: 1956e509be898d8937634437caa474b57ac12795e5bcRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 195712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivannack_acc: 195853dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell ipath_rc_error(qp, IB_WC_LOC_PROT_ERR); 19593859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; 19603859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_ack_psn = qp->r_psn; 196112eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivansend_ack: 19623859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell send_rc_ack(qp); 1963e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto done; 196497f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan 1965e509be898d8937634437caa474b57ac12795e5bcRalph Campbellunlock: 1966e509be898d8937634437caa474b57ac12795e5bcRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 196712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivandone: 196897f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan return; 196997f9efbc47f0b1bc88abac8724b505f0794a48d0Bryan O'Sullivan} 1970