174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan/* 2e509be898d8937634437caa474b57ac12795e5bcRalph Campbell * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved. 374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. 474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * 574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * This software is available to you under a choice of one of two 674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * licenses. You may choose to be licensed under the terms of the GNU 774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * General Public License (GPL) Version 2, available from the file 874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * COPYING in the main directory of this source tree, or the 974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * OpenIB.org BSD license below: 1074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * 1174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * Redistribution and use in source and binary forms, with or 1274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * without modification, are permitted provided that the following 1374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * conditions are met: 1474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * 1574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * - Redistributions of source code must retain the above 1674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * copyright notice, this list of conditions and the following 1774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * disclaimer. 1874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * 1974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * - Redistributions in binary form must reproduce the above 2074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * copyright notice, this list of conditions and the following 2174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * disclaimer in the documentation and/or other materials 2274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * provided with the distribution. 2374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * 2474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 2574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 2674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 2774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 2874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 2974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 3074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 3174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * SOFTWARE. 3274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan */ 3374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 3474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan#include "ipath_verbs.h" 3534b2aafea38efdf02cd8107a6e1057e2a297c447Bryan O'Sullivan#include "ipath_kernel.h" 3674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 3774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan/* cut down ridiculously long IB macro names */ 3874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan#define OP(x) IB_OPCODE_UC_##x 3974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 4074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan/** 41ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * ipath_make_uc_req - construct a request packet (SEND, RDMA write) 42ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * @qp: a pointer to the QP 4374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * 44ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * Return 1 if constructed; otherwise, return 0. 4574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan */ 464ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbellint ipath_make_uc_req(struct ipath_qp *qp) 4774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan{ 484ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell struct ipath_other_headers *ohdr; 4974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan struct ipath_swqe *wqe; 50e509be898d8937634437caa474b57ac12795e5bcRalph Campbell unsigned long flags; 5174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan u32 hwords; 5274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan u32 bth0; 5374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan u32 len; 544ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu); 554ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell int ret = 0; 5674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 57e509be898d8937634437caa474b57ac12795e5bcRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 58e509be898d8937634437caa474b57ac12795e5bcRalph Campbell 59e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK)) { 60e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (!(ib_ipath_state_ops[qp->state] & IPATH_FLUSH_SEND)) 61e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto bail; 62e509be898d8937634437caa474b57ac12795e5bcRalph Campbell /* We are in the error state, flush the work request. */ 63e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (qp->s_last == qp->s_head) 64e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto bail; 65e509be898d8937634437caa474b57ac12795e5bcRalph Campbell /* If DMAs are in progress, we can't flush immediately. */ 66e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (atomic_read(&qp->s_dma_busy)) { 67e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->s_flags |= IPATH_S_WAIT_DMA; 68e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto bail; 69e509be898d8937634437caa474b57ac12795e5bcRalph Campbell } 70e509be898d8937634437caa474b57ac12795e5bcRalph Campbell wqe = get_swqe_ptr(qp, qp->s_last); 71e509be898d8937634437caa474b57ac12795e5bcRalph Campbell ipath_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); 7274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto done; 73e509be898d8937634437caa474b57ac12795e5bcRalph Campbell } 7474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 754ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell ohdr = &qp->s_hdr.u.oth; 764ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell if (qp->remote_ah_attr.ah_flags & IB_AH_GRH) 774ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell ohdr = &qp->s_hdr.u.l.oth; 784ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell 79ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan /* header size in 32-bit words LRH+BTH = (8+12)/4. */ 80ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hwords = 5; 811dd6a1be1416be48cafda9e63a614f26f0428d10Ralph Campbell bth0 = 1 << 22; /* Set M bit */ 8274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 83ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan /* Get the next send request. */ 844ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell wqe = get_swqe_ptr(qp, qp->s_cur); 854ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp->s_wqe = NULL; 8674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan switch (qp->s_state) { 8774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan default: 88e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (!(ib_ipath_state_ops[qp->state] & 89e509be898d8937634437caa474b57ac12795e5bcRalph Campbell IPATH_PROCESS_NEXT_SEND_OK)) 90e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto bail; 9174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Check if send work queue is empty. */ 924ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell if (qp->s_cur == qp->s_head) 93e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto bail; 9474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* 9574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * Start a new request. 9674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan */ 9774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->s_psn = wqe->psn = qp->s_next_psn; 9874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->s_sge.sge = wqe->sg_list[0]; 9974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->s_sge.sg_list = wqe->sg_list + 1; 10074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->s_sge.num_sge = wqe->wr.num_sge; 10174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->s_len = len = wqe->length; 10274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan switch (wqe->wr.opcode) { 10374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case IB_WR_SEND: 10474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case IB_WR_SEND_WITH_IMM: 10574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (len > pmtu) { 10674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->s_state = OP(SEND_FIRST); 10774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan len = pmtu; 10874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan break; 10974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 11074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_SEND) 11174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->s_state = OP(SEND_ONLY); 11274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan else { 11374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->s_state = 11474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan OP(SEND_ONLY_WITH_IMMEDIATE); 11574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Immediate data comes after the BTH */ 1160f39cf3d54e67a705773fd0ec56ca3dcd3e9272fRoland Dreier ohdr->u.imm_data = wqe->wr.ex.imm_data; 11774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan hwords += 1; 11874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 11974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (wqe->wr.send_flags & IB_SEND_SOLICITED) 12074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan bth0 |= 1 << 23; 1214ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp->s_wqe = wqe; 1224ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell if (++qp->s_cur >= qp->s_size) 1234ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp->s_cur = 0; 12474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan break; 12574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 12674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case IB_WR_RDMA_WRITE: 12774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case IB_WR_RDMA_WRITE_WITH_IMM: 12874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan ohdr->u.rc.reth.vaddr = 12974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan cpu_to_be64(wqe->wr.wr.rdma.remote_addr); 13074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan ohdr->u.rc.reth.rkey = 13174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan cpu_to_be32(wqe->wr.wr.rdma.rkey); 13274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan ohdr->u.rc.reth.length = cpu_to_be32(len); 13374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan hwords += sizeof(struct ib_reth) / 4; 13474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (len > pmtu) { 13574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->s_state = OP(RDMA_WRITE_FIRST); 13674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan len = pmtu; 13774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan break; 13874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 13974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_RDMA_WRITE) 14074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->s_state = OP(RDMA_WRITE_ONLY); 14174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan else { 14274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->s_state = 14374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE); 14474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Immediate data comes after the RETH */ 1450f39cf3d54e67a705773fd0ec56ca3dcd3e9272fRoland Dreier ohdr->u.rc.imm_data = wqe->wr.ex.imm_data; 14674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan hwords += 1; 14774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (wqe->wr.send_flags & IB_SEND_SOLICITED) 14874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan bth0 |= 1 << 23; 14974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 1504ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp->s_wqe = wqe; 1514ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell if (++qp->s_cur >= qp->s_size) 1524ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp->s_cur = 0; 15374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan break; 15474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 15574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan default: 156e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto bail; 15774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 15874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan break; 15974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 16074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(SEND_FIRST): 16174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->s_state = OP(SEND_MIDDLE); 16274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* FALLTHROUGH */ 16374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(SEND_MIDDLE): 16474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan len = qp->s_len; 16574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (len > pmtu) { 16674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan len = pmtu; 16774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan break; 16874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 16974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_SEND) 17074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->s_state = OP(SEND_LAST); 17174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan else { 17274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); 17374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Immediate data comes after the BTH */ 1740f39cf3d54e67a705773fd0ec56ca3dcd3e9272fRoland Dreier ohdr->u.imm_data = wqe->wr.ex.imm_data; 17574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan hwords += 1; 17674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 17774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (wqe->wr.send_flags & IB_SEND_SOLICITED) 17874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan bth0 |= 1 << 23; 1794ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp->s_wqe = wqe; 1804ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell if (++qp->s_cur >= qp->s_size) 1814ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp->s_cur = 0; 18274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan break; 18374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 18474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(RDMA_WRITE_FIRST): 18574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->s_state = OP(RDMA_WRITE_MIDDLE); 18674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* FALLTHROUGH */ 18774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(RDMA_WRITE_MIDDLE): 18874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan len = qp->s_len; 18974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (len > pmtu) { 19074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan len = pmtu; 19174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan break; 19274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 19374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_RDMA_WRITE) 19474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->s_state = OP(RDMA_WRITE_LAST); 19574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan else { 19674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->s_state = 19774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); 19874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Immediate data comes after the BTH */ 1990f39cf3d54e67a705773fd0ec56ca3dcd3e9272fRoland Dreier ohdr->u.imm_data = wqe->wr.ex.imm_data; 20074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan hwords += 1; 20174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (wqe->wr.send_flags & IB_SEND_SOLICITED) 20274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan bth0 |= 1 << 23; 20374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 2044ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp->s_wqe = wqe; 2054ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell if (++qp->s_cur >= qp->s_size) 2064ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp->s_cur = 0; 20774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan break; 20874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 20974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->s_len -= len; 21074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->s_hdrwords = hwords; 21174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->s_cur_sge = &qp->s_sge; 21274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->s_cur_size = len; 2134ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell ipath_make_ruc_header(to_idev(qp->ibqp.device), 2144ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp, ohdr, bth0 | (qp->s_state << 24), 2154ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp->s_next_psn++ & IPATH_PSN_MASK); 216e509be898d8937634437caa474b57ac12795e5bcRalph Campbelldone: 2174ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell ret = 1; 218e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto unlock; 21974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 220e509be898d8937634437caa474b57ac12795e5bcRalph Campbellbail: 221e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->s_flags &= ~IPATH_S_BUSY; 222e509be898d8937634437caa474b57ac12795e5bcRalph Campbellunlock: 223e509be898d8937634437caa474b57ac12795e5bcRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 2244ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell return ret; 22574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan} 22674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 22774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan/** 22874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * ipath_uc_rcv - handle an incoming UC packet 22974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * @dev: the device the packet came in on 23074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * @hdr: the header of the packet 23174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * @has_grh: true if the packet has a GRH 23274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * @data: the packet data 23374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * @tlen: the length of the packet 23474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * @qp: the QP for this packet. 23574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * 23674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * This is called from ipath_qp_rcv() to process an incoming UC packet 23774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * for the given QP. 23874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * Called at interrupt level. 23974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan */ 24074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivanvoid ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 24174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan int has_grh, void *data, u32 tlen, struct ipath_qp *qp) 24274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan{ 24374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan struct ipath_other_headers *ohdr; 24474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan int opcode; 24574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan u32 hdrsize; 24674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan u32 psn; 24774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan u32 pad; 24874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan struct ib_wc wc; 24974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu); 25074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan struct ib_reth *reth; 25174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan int header_in_data; 25274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 25310aeb0e6d8823c1cccf9edc8401c848745c128beBryan O'Sullivan /* Validate the SLID. See Ch. 9.6.1.5 */ 25410aeb0e6d8823c1cccf9edc8401c848745c128beBryan O'Sullivan if (unlikely(be16_to_cpu(hdr->lrh[3]) != qp->remote_ah_attr.dlid)) 25510aeb0e6d8823c1cccf9edc8401c848745c128beBryan O'Sullivan goto done; 25610aeb0e6d8823c1cccf9edc8401c848745c128beBryan O'Sullivan 25774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Check for GRH */ 25874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (!has_grh) { 25974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan ohdr = &hdr->u.oth; 26074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan hdrsize = 8 + 12; /* LRH + BTH */ 26174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan psn = be32_to_cpu(ohdr->bth[2]); 26274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan header_in_data = 0; 26374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } else { 26474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan ohdr = &hdr->u.l.oth; 26574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan hdrsize = 8 + 40 + 12; /* LRH + GRH + BTH */ 26674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* 26774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * The header with GRH is 60 bytes and the 26874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * core driver sets the eager header buffer 26974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * size to 56 bytes so the last 4 bytes of 27074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * the BTH header (PSN) is in the data buffer. 27174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan */ 27234b2aafea38efdf02cd8107a6e1057e2a297c447Bryan O'Sullivan header_in_data = dev->dd->ipath_rcvhdrentsize == 16; 27374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (header_in_data) { 27474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan psn = be32_to_cpu(((__be32 *) data)[0]); 27574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan data += sizeof(__be32); 27674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } else 27774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan psn = be32_to_cpu(ohdr->bth[2]); 27874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 27974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* 28074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * The opcode is in the low byte when its in network order 28174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * (top byte when in host order). 28274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan */ 28374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan opcode = be32_to_cpu(ohdr->bth[0]) >> 24; 28474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 285e509be898d8937634437caa474b57ac12795e5bcRalph Campbell memset(&wc, 0, sizeof wc); 28674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 28774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Compare the PSN verses the expected PSN. */ 28874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (unlikely(ipath_cmp24(psn, qp->r_psn) != 0)) { 28974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* 29074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * Handle a sequence error. 29174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan * Silently drop any current message. 29274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan */ 29374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->r_psn = psn; 29474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan inv: 29574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->r_state = OP(SEND_LAST); 29674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan switch (opcode) { 29774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(SEND_FIRST): 29874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(SEND_ONLY): 29974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(SEND_ONLY_WITH_IMMEDIATE): 30074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto send_first; 30174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 30274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(RDMA_WRITE_FIRST): 30374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(RDMA_WRITE_ONLY): 30474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): 30574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto rdma_first; 30674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 30774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan default: 30874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan dev->n_pkt_drops++; 30974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto done; 31074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 31174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 31274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 31374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Check for opcode sequence errors. */ 31474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan switch (qp->r_state) { 31574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(SEND_FIRST): 31674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(SEND_MIDDLE): 31774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (opcode == OP(SEND_MIDDLE) || 31874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan opcode == OP(SEND_LAST) || 31974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan opcode == OP(SEND_LAST_WITH_IMMEDIATE)) 32074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan break; 32174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto inv; 32274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 32374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(RDMA_WRITE_FIRST): 32474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(RDMA_WRITE_MIDDLE): 32574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (opcode == OP(RDMA_WRITE_MIDDLE) || 32674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan opcode == OP(RDMA_WRITE_LAST) || 32774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) 32874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan break; 32974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto inv; 33074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 33174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan default: 33274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (opcode == OP(SEND_FIRST) || 33374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan opcode == OP(SEND_ONLY) || 33474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan opcode == OP(SEND_ONLY_WITH_IMMEDIATE) || 33574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan opcode == OP(RDMA_WRITE_FIRST) || 33674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan opcode == OP(RDMA_WRITE_ONLY) || 33774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE)) 33874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan break; 33974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto inv; 34074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 34174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 34274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* OK, process the packet. */ 34374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan switch (opcode) { 34474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(SEND_FIRST): 34574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(SEND_ONLY): 34674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(SEND_ONLY_WITH_IMMEDIATE): 34774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan send_first: 348e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (qp->r_flags & IPATH_R_REUSE_SGE) { 349e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->r_flags &= ~IPATH_R_REUSE_SGE; 3503859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->r_sge = qp->s_rdma_read_sge; 35174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } else if (!ipath_get_rwqe(qp, 0)) { 35274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan dev->n_pkt_drops++; 35374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto done; 35474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 35574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Save the WQE so we can reuse it in case of an error. */ 3563859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell qp->s_rdma_read_sge = qp->r_sge; 35774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->r_rcv_len = 0; 35874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (opcode == OP(SEND_ONLY)) 35974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto send_last; 36074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan else if (opcode == OP(SEND_ONLY_WITH_IMMEDIATE)) 36174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto send_last_imm; 36274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* FALLTHROUGH */ 36374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(SEND_MIDDLE): 36474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Check for invalid length PMTU or posted rwqe len. */ 36574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (unlikely(tlen != (hdrsize + pmtu + 4))) { 366e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->r_flags |= IPATH_R_REUSE_SGE; 36774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan dev->n_pkt_drops++; 36874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto done; 36974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 37074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->r_rcv_len += pmtu; 37174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (unlikely(qp->r_rcv_len > qp->r_len)) { 372e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->r_flags |= IPATH_R_REUSE_SGE; 37374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan dev->n_pkt_drops++; 37474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto done; 37574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 37674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan ipath_copy_sge(&qp->r_sge, data, pmtu); 37774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan break; 37874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 37974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(SEND_LAST_WITH_IMMEDIATE): 38074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan send_last_imm: 38174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (header_in_data) { 38200f7ec36c9324928e4cd23f02e6d8550f30c32caSteve Wise wc.ex.imm_data = *(__be32 *) data; 38374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan data += sizeof(__be32); 38474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } else { 38574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Immediate data comes after BTH */ 38600f7ec36c9324928e4cd23f02e6d8550f30c32caSteve Wise wc.ex.imm_data = ohdr->u.imm_data; 38774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 38874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan hdrsize += 4; 38974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan wc.wc_flags = IB_WC_WITH_IMM; 39074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* FALLTHROUGH */ 39174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(SEND_LAST): 39274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan send_last: 39374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Get the number of bytes the message was padded by. */ 39474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 39574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Check for invalid length. */ 39674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* XXX LAST len should be >= 1 */ 39774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (unlikely(tlen < (hdrsize + pad + 4))) { 398e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->r_flags |= IPATH_R_REUSE_SGE; 39974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan dev->n_pkt_drops++; 40074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto done; 40174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 40274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Don't count the CRC. */ 40374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan tlen -= (hdrsize + pad + 4); 40474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan wc.byte_len = tlen + qp->r_rcv_len; 40574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (unlikely(wc.byte_len > qp->r_len)) { 406e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->r_flags |= IPATH_R_REUSE_SGE; 40774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan dev->n_pkt_drops++; 40874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto done; 40974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 410df3f0da8db6b5e7e8f0585221c8b1cd8ff806d35Ralph Campbell wc.opcode = IB_WC_RECV; 41174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan last_imm: 41274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan ipath_copy_sge(&qp->r_sge, data, tlen); 41374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan wc.wr_id = qp->r_wr_id; 41474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan wc.status = IB_WC_SUCCESS; 415062dbb69f32b9ccea701b30f8cc0049482e6211fMichael S. Tsirkin wc.qp = &qp->ibqp; 41674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan wc.src_qp = qp->remote_qpn; 41774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan wc.slid = qp->remote_ah_attr.dlid; 41874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan wc.sl = qp->remote_ah_attr.sl; 41974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Signal completion event if the solicited bit is set. */ 42074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 42174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan (ohdr->bth[0] & 4229c3da0991754d480328eeaa2b90cb231a1cea9b6Harvey Harrison cpu_to_be32(1 << 23)) != 0); 42374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan break; 42474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 42574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(RDMA_WRITE_FIRST): 42674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(RDMA_WRITE_ONLY): 42774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): /* consume RWQE */ 42874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan rdma_first: 42974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* RETH comes after BTH */ 43074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (!header_in_data) 43174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan reth = &ohdr->u.rc.reth; 43274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan else { 43374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan reth = (struct ib_reth *)data; 43474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan data += sizeof(*reth); 43574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 43674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan hdrsize += sizeof(*reth); 43774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->r_len = be32_to_cpu(reth->length); 43874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->r_rcv_len = 0; 43974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (qp->r_len != 0) { 44074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan u32 rkey = be32_to_cpu(reth->rkey); 44174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan u64 vaddr = be64_to_cpu(reth->vaddr); 442ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan int ok; 44374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 44474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Check rkey */ 4456a553af286653818bb5831f1b351eefdc8a93b61Bryan O'Sullivan ok = ipath_rkey_ok(qp, &qp->r_sge, qp->r_len, 446ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan vaddr, rkey, 447ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan IB_ACCESS_REMOTE_WRITE); 448ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan if (unlikely(!ok)) { 44974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan dev->n_pkt_drops++; 45074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto done; 45174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 45274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } else { 45374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->r_sge.sg_list = NULL; 45474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->r_sge.sge.mr = NULL; 45574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->r_sge.sge.vaddr = NULL; 45674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->r_sge.sge.length = 0; 45774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->r_sge.sge.sge_length = 0; 45874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 45974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (unlikely(!(qp->qp_access_flags & 46074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan IB_ACCESS_REMOTE_WRITE))) { 46174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan dev->n_pkt_drops++; 46274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto done; 46374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 46474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (opcode == OP(RDMA_WRITE_ONLY)) 46574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto rdma_last; 466ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan else if (opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE)) 46774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto rdma_last_imm; 46874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* FALLTHROUGH */ 46974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(RDMA_WRITE_MIDDLE): 47074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Check for invalid length PMTU or posted rwqe len. */ 47174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (unlikely(tlen != (hdrsize + pmtu + 4))) { 47274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan dev->n_pkt_drops++; 47374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto done; 47474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 47574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->r_rcv_len += pmtu; 47674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (unlikely(qp->r_rcv_len > qp->r_len)) { 47774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan dev->n_pkt_drops++; 47874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto done; 47974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 48074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan ipath_copy_sge(&qp->r_sge, data, pmtu); 48174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan break; 48274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 48374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE): 48474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan rdma_last_imm: 48555046698faf6357ff7c53593dbfd43a9a3f681a7Ralph Campbell if (header_in_data) { 48600f7ec36c9324928e4cd23f02e6d8550f30c32caSteve Wise wc.ex.imm_data = *(__be32 *) data; 48755046698faf6357ff7c53593dbfd43a9a3f681a7Ralph Campbell data += sizeof(__be32); 48855046698faf6357ff7c53593dbfd43a9a3f681a7Ralph Campbell } else { 48955046698faf6357ff7c53593dbfd43a9a3f681a7Ralph Campbell /* Immediate data comes after BTH */ 49000f7ec36c9324928e4cd23f02e6d8550f30c32caSteve Wise wc.ex.imm_data = ohdr->u.imm_data; 49155046698faf6357ff7c53593dbfd43a9a3f681a7Ralph Campbell } 49255046698faf6357ff7c53593dbfd43a9a3f681a7Ralph Campbell hdrsize += 4; 49355046698faf6357ff7c53593dbfd43a9a3f681a7Ralph Campbell wc.wc_flags = IB_WC_WITH_IMM; 49455046698faf6357ff7c53593dbfd43a9a3f681a7Ralph Campbell 49574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Get the number of bytes the message was padded by. */ 49674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 49774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Check for invalid length. */ 49874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* XXX LAST len should be >= 1 */ 49974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (unlikely(tlen < (hdrsize + pad + 4))) { 50074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan dev->n_pkt_drops++; 50174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto done; 50274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 50374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Don't count the CRC. */ 50474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan tlen -= (hdrsize + pad + 4); 50574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (unlikely(tlen + qp->r_rcv_len != qp->r_len)) { 50674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan dev->n_pkt_drops++; 50774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto done; 50874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 509e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (qp->r_flags & IPATH_R_REUSE_SGE) 510e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->r_flags &= ~IPATH_R_REUSE_SGE; 511ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan else if (!ipath_get_rwqe(qp, 1)) { 51274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan dev->n_pkt_drops++; 51374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto done; 51474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 51555046698faf6357ff7c53593dbfd43a9a3f681a7Ralph Campbell wc.byte_len = qp->r_len; 516df3f0da8db6b5e7e8f0585221c8b1cd8ff806d35Ralph Campbell wc.opcode = IB_WC_RECV_RDMA_WITH_IMM; 51774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto last_imm; 51874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 51974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan case OP(RDMA_WRITE_LAST): 52074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan rdma_last: 52174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Get the number of bytes the message was padded by. */ 52274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 52374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Check for invalid length. */ 52474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* XXX LAST len should be >= 1 */ 52574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (unlikely(tlen < (hdrsize + pad + 4))) { 52674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan dev->n_pkt_drops++; 52774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto done; 52874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 52974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Don't count the CRC. */ 53074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan tlen -= (hdrsize + pad + 4); 53174ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan if (unlikely(tlen + qp->r_rcv_len != qp->r_len)) { 53274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan dev->n_pkt_drops++; 53374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan goto done; 53474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 53574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan ipath_copy_sge(&qp->r_sge, data, tlen); 53674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan break; 53774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan 53874ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan default: 53974ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan /* Drop packet for unknown opcodes. */ 54074ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan dev->n_pkt_drops++; 54112eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto done; 54274ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan } 54374ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->r_psn++; 54474ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan qp->r_state = opcode; 54574ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivandone: 54674ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan return; 54774ed6b5eb133b4acae7c47bc23457e5f8e7c1125Bryan O'Sullivan} 548