1e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan/* 253dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved. 3e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. 4e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * 5e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * This software is available to you under a choice of one of two 6e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * licenses. You may choose to be licensed under the terms of the GNU 7e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * General Public License (GPL) Version 2, available from the file 8e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * COPYING in the main directory of this source tree, or the 9e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * OpenIB.org BSD license below: 10e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * 11e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * Redistribution and use in source and binary forms, with or 12e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * without modification, are permitted provided that the following 13e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * conditions are met: 14e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * 15e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * - Redistributions of source code must retain the above 16e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * copyright notice, this list of conditions and the following 17e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * disclaimer. 18e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * 19e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * - Redistributions in binary form must reproduce the above 20e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * copyright notice, this list of conditions and the following 21e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * disclaimer in the documentation and/or other materials 22e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * provided with the distribution. 23e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * 24e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * SOFTWARE. 32e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan */ 33e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 34d43c36dc6b357fa1806800f18aa30123c747a6d1Alexey Dobriyan#include <linux/sched.h> 354ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell#include <linux/spinlock.h> 364ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell 37e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan#include "ipath_verbs.h" 38373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell#include "ipath_kernel.h" 39e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 40e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan/* 41e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * Convert the AETH RNR timeout code into the number of milliseconds. 42e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan */ 43e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivanconst u32 ib_ipath_rnr_table[32] = { 44e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 656, /* 0 */ 45e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* 1 */ 46e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* 2 */ 47e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* 3 */ 48e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* 4 */ 49e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* 5 */ 50e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* 6 */ 51e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* 7 */ 52e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* 8 */ 53e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* 9 */ 54e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* A */ 55e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* B */ 56e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* C */ 57e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* D */ 58e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 2, /* E */ 59e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 2, /* F */ 60e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 3, /* 10 */ 61e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 4, /* 11 */ 62e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 6, /* 12 */ 63e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 8, /* 13 */ 64e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 11, /* 14 */ 65e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 16, /* 15 */ 66e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 21, /* 16 */ 67e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 31, /* 17 */ 68e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 41, /* 18 */ 69e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 62, /* 19 */ 70e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 82, /* 1A */ 71e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 123, /* 1B */ 72e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 164, /* 1C */ 73e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 246, /* 1D */ 74e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 328, /* 1E */ 75e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 492 /* 1F */ 76e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan}; 77e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 78e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan/** 79e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * ipath_insert_rnr_queue - put QP on the RNR timeout list for the device 80e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * @qp: the QP 81e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * 82e509be898d8937634437caa474b57ac12795e5bcRalph Campbell * Called with the QP s_lock held and interrupts disabled. 83e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * XXX Use a simple list for now. We might need a priority 84e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * queue if we have lots of QPs waiting for RNR timeouts 85e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * but that should be rare. 86e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan */ 87e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivanvoid ipath_insert_rnr_queue(struct ipath_qp *qp) 88e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan{ 89e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct ipath_ibdev *dev = to_idev(qp->ibqp.device); 90e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 91e509be898d8937634437caa474b57ac12795e5bcRalph Campbell /* We already did a spin_lock_irqsave(), so just use spin_lock */ 92e509be898d8937634437caa474b57ac12795e5bcRalph Campbell spin_lock(&dev->pending_lock); 93e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (list_empty(&dev->rnrwait)) 94e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan list_add(&qp->timerwait, &dev->rnrwait); 95e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan else { 96e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct list_head *l = &dev->rnrwait; 97e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct ipath_qp *nqp = list_entry(l->next, struct ipath_qp, 98e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan timerwait); 99e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 100e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan while (qp->s_rnr_timeout >= nqp->s_rnr_timeout) { 101e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->s_rnr_timeout -= nqp->s_rnr_timeout; 102e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan l = l->next; 103cc65edcf0c174eff4367cfbc594a2f33c0d477faRalph Campbell if (l->next == &dev->rnrwait) { 104cc65edcf0c174eff4367cfbc594a2f33c0d477faRalph Campbell nqp = NULL; 105e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan break; 106cc65edcf0c174eff4367cfbc594a2f33c0d477faRalph Campbell } 107e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan nqp = list_entry(l->next, struct ipath_qp, 108e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan timerwait); 109e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 110cc65edcf0c174eff4367cfbc594a2f33c0d477faRalph Campbell if (nqp) 111cc65edcf0c174eff4367cfbc594a2f33c0d477faRalph Campbell nqp->s_rnr_timeout -= qp->s_rnr_timeout; 112e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan list_add(&qp->timerwait, l); 113e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 114e509be898d8937634437caa474b57ac12795e5bcRalph Campbell spin_unlock(&dev->pending_lock); 115e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan} 116e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1174ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell/** 1184ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell * ipath_init_sge - Validate a RWQE and fill in the SGE state 1194ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell * @qp: the QP 1204ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell * 1214ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell * Return 1 if OK. 1224ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell */ 1234ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbellint ipath_init_sge(struct ipath_qp *qp, struct ipath_rwqe *wqe, 1244ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell u32 *lengthp, struct ipath_sge_state *ss) 125373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell{ 126373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell int i, j, ret; 127373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell struct ib_wc wc; 128373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell 1294ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell *lengthp = 0; 130373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell for (i = j = 0; i < wqe->num_sge; i++) { 131373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if (wqe->sg_list[i].length == 0) 132373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell continue; 133373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell /* Check LKEY */ 1344ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell if (!ipath_lkey_ok(qp, j ? &ss->sg_list[j - 1] : &ss->sge, 1354ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell &wqe->sg_list[i], IB_ACCESS_LOCAL_WRITE)) 136373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell goto bad_lkey; 1374ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell *lengthp += wqe->sg_list[i].length; 138373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell j++; 139373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell } 1404ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell ss->num_sge = j; 141373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell ret = 1; 142373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell goto bail; 143373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell 144373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbellbad_lkey: 14553dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell memset(&wc, 0, sizeof(wc)); 146373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell wc.wr_id = wqe->wr_id; 147373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell wc.status = IB_WC_LOC_PROT_ERR; 148373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell wc.opcode = IB_WC_RECV; 149062dbb69f32b9ccea701b30f8cc0049482e6211fMichael S. Tsirkin wc.qp = &qp->ibqp; 150373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell /* Signal solicited completion event. */ 151373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1); 152373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell ret = 0; 153373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbellbail: 154373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell return ret; 155373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell} 156373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell 157e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan/** 158e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * ipath_get_rwqe - copy the next RWQE into the QP's RWQE 159e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * @qp: the QP 160a8b56f296d7d977fea2512e353a131f8da490aa5Ralph Campbell * @wr_id_only: update qp->r_wr_id only, not qp->r_sge 161e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * 162e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * Return 0 if no RWQE is available, otherwise return 1. 163e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * 16412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan * Can be called from interrupt level. 165e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan */ 166e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivanint ipath_get_rwqe(struct ipath_qp *qp, int wr_id_only) 167e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan{ 16812eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan unsigned long flags; 169e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct ipath_rq *rq; 170373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell struct ipath_rwq *wq; 171e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct ipath_srq *srq; 172e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct ipath_rwqe *wqe; 173373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell void (*handler)(struct ib_event *, void *); 174373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell u32 tail; 175373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell int ret; 176e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 177373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if (qp->ibqp.srq) { 178373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell srq = to_isrq(qp->ibqp.srq); 179373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell handler = srq->ibsrq.event_handler; 180373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell rq = &srq->rq; 181373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell } else { 182373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell srq = NULL; 183373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell handler = NULL; 184e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan rq = &qp->r_rq; 185e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 186e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 18712eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan spin_lock_irqsave(&rq->lock, flags); 188e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) { 189e509be898d8937634437caa474b57ac12795e5bcRalph Campbell ret = 0; 190e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto unlock; 191e509be898d8937634437caa474b57ac12795e5bcRalph Campbell } 192e509be898d8937634437caa474b57ac12795e5bcRalph Campbell 193373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell wq = rq->wq; 194373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell tail = wq->tail; 195373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell /* Validate tail before using it since it is user writable. */ 196373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if (tail >= rq->size) 197373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell tail = 0; 198373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell do { 199373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if (unlikely(tail == wq->head)) { 200373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell ret = 0; 201e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto unlock; 202373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell } 2034fc570bcbe77f823aae183dd824869f79e74cc97Ralph Campbell /* Make sure entry is read after head index is read. */ 2044fc570bcbe77f823aae183dd824869f79e74cc97Ralph Campbell smp_rmb(); 205373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell wqe = get_rwqe_ptr(rq, tail); 206373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if (++tail >= rq->size) 207373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell tail = 0; 208a8b56f296d7d977fea2512e353a131f8da490aa5Ralph Campbell if (wr_id_only) 209a8b56f296d7d977fea2512e353a131f8da490aa5Ralph Campbell break; 210a8b56f296d7d977fea2512e353a131f8da490aa5Ralph Campbell qp->r_sge.sg_list = qp->r_sg_list; 211a8b56f296d7d977fea2512e353a131f8da490aa5Ralph Campbell } while (!ipath_init_sge(qp, wqe, &qp->r_len, &qp->r_sge)); 212e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->r_wr_id = wqe->wr_id; 213373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell wq->tail = tail; 214373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell 215373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell ret = 1; 216e509be898d8937634437caa474b57ac12795e5bcRalph Campbell set_bit(IPATH_R_WRID_VALID, &qp->r_aflags); 217373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if (handler) { 218e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan u32 n; 219e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 220373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell /* 221373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell * validate head pointer value and compute 222373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell * the number of remaining WQEs. 223373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell */ 224373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell n = wq->head; 225373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if (n >= rq->size) 226373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell n = 0; 227373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell if (n < tail) 228373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell n += rq->size - tail; 229e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan else 230373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell n -= tail; 231e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (n < srq->limit) { 232373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell struct ib_event ev; 233373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell 234e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan srq->limit = 0; 23512eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan spin_unlock_irqrestore(&rq->lock, flags); 236e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ev.device = qp->ibqp.device; 237e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ev.element.srq = qp->ibqp.srq; 238e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ev.event = IB_EVENT_SRQ_LIMIT_REACHED; 239373d9915803aebbbf7fd3841efd9dac31c32e148Ralph Campbell handler(&ev, srq->ibsrq.srq_context); 24012eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan goto bail; 24112eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan } 24212eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan } 243e509be898d8937634437caa474b57ac12795e5bcRalph Campbellunlock: 24412eef41f8b72b6e11e36b48c78849c17e49781c8Bryan O'Sullivan spin_unlock_irqrestore(&rq->lock, flags); 245e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivanbail: 246e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan return ret; 247e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan} 248e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 249e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan/** 250e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * ipath_ruc_loopback - handle UC and RC lookback requests 2514ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell * @sqp: the sending QP 252e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * 2534ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell * This is called from ipath_do_send() to 254e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * forward a WQE addressed to the same HCA. 255e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * Note that although we are single threaded due to the tasklet, we still 256e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * have to protect against post_send(). We don't have to worry about 257e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * receive interrupts since this is a connected protocol and all packets 258e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * will pass through here. 259e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan */ 260ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivanstatic void ipath_ruc_loopback(struct ipath_qp *sqp) 261e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan{ 262e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct ipath_ibdev *dev = to_idev(sqp->ibqp.device); 263e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct ipath_qp *qp; 264e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct ipath_swqe *wqe; 265e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct ipath_sge *sge; 266e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan unsigned long flags; 267ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan struct ib_wc wc; 268e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan u64 sdata; 2693859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell atomic64_t *maddr; 27053dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell enum ib_wc_status send_status; 271e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 272e509be898d8937634437caa474b57ac12795e5bcRalph Campbell /* 273e509be898d8937634437caa474b57ac12795e5bcRalph Campbell * Note that we check the responder QP state after 274e509be898d8937634437caa474b57ac12795e5bcRalph Campbell * checking the requester's state. 275e509be898d8937634437caa474b57ac12795e5bcRalph Campbell */ 276e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp = ipath_lookup_qpn(&dev->qp_table, sqp->remote_qpn); 277e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 278e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_lock_irqsave(&sqp->s_lock, flags); 279e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 280e509be898d8937634437caa474b57ac12795e5bcRalph Campbell /* Return if we are already busy processing a work request. */ 281e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if ((sqp->s_flags & (IPATH_S_BUSY | IPATH_S_ANY_WAIT)) || 282e509be898d8937634437caa474b57ac12795e5bcRalph Campbell !(ib_ipath_state_ops[sqp->state] & IPATH_PROCESS_OR_FLUSH_SEND)) 283e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto unlock; 284e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 285e509be898d8937634437caa474b57ac12795e5bcRalph Campbell sqp->s_flags |= IPATH_S_BUSY; 286e509be898d8937634437caa474b57ac12795e5bcRalph Campbell 287e509be898d8937634437caa474b57ac12795e5bcRalph Campbellagain: 288e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (sqp->s_last == sqp->s_head) 289e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto clr_busy; 290e509be898d8937634437caa474b57ac12795e5bcRalph Campbell wqe = get_swqe_ptr(sqp, sqp->s_last); 291e509be898d8937634437caa474b57ac12795e5bcRalph Campbell 292e509be898d8937634437caa474b57ac12795e5bcRalph Campbell /* Return if it is not OK to start a new work reqeust. */ 293e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (!(ib_ipath_state_ops[sqp->state] & IPATH_PROCESS_NEXT_SEND_OK)) { 294e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (!(ib_ipath_state_ops[sqp->state] & IPATH_FLUSH_SEND)) 295e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto clr_busy; 296e509be898d8937634437caa474b57ac12795e5bcRalph Campbell /* We are in the error state, flush the work request. */ 297e509be898d8937634437caa474b57ac12795e5bcRalph Campbell send_status = IB_WC_WR_FLUSH_ERR; 298e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto flush_send; 299e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 300e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 301e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan /* 302e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * We can rely on the entry not changing without the s_lock 303e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * being held until we update s_last. 304e509be898d8937634437caa474b57ac12795e5bcRalph Campbell * We increment s_cur to indicate s_last is in progress. 305e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan */ 306e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (sqp->s_last == sqp->s_cur) { 307e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (++sqp->s_cur >= sqp->s_size) 308e509be898d8937634437caa474b57ac12795e5bcRalph Campbell sqp->s_cur = 0; 309e509be898d8937634437caa474b57ac12795e5bcRalph Campbell } 310e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_unlock_irqrestore(&sqp->s_lock, flags); 311e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 312e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (!qp || !(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) { 313e509be898d8937634437caa474b57ac12795e5bcRalph Campbell dev->n_pkt_drops++; 314e509be898d8937634437caa474b57ac12795e5bcRalph Campbell /* 315e509be898d8937634437caa474b57ac12795e5bcRalph Campbell * For RC, the requester would timeout and retry so 316e509be898d8937634437caa474b57ac12795e5bcRalph Campbell * shortcut the timeouts and just signal too many retries. 317e509be898d8937634437caa474b57ac12795e5bcRalph Campbell */ 318e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (sqp->ibqp.qp_type == IB_QPT_RC) 319e509be898d8937634437caa474b57ac12795e5bcRalph Campbell send_status = IB_WC_RETRY_EXC_ERR; 320e509be898d8937634437caa474b57ac12795e5bcRalph Campbell else 321e509be898d8937634437caa474b57ac12795e5bcRalph Campbell send_status = IB_WC_SUCCESS; 322e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto serr; 323e509be898d8937634437caa474b57ac12795e5bcRalph Campbell } 324e509be898d8937634437caa474b57ac12795e5bcRalph Campbell 32553dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell memset(&wc, 0, sizeof wc); 32653dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell send_status = IB_WC_SUCCESS; 327e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 328e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sqp->s_sge.sge = wqe->sg_list[0]; 329e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sqp->s_sge.sg_list = wqe->sg_list + 1; 330e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sqp->s_sge.num_sge = wqe->wr.num_sge; 331e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sqp->s_len = wqe->length; 332e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan switch (wqe->wr.opcode) { 333e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan case IB_WR_SEND_WITH_IMM: 334ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan wc.wc_flags = IB_WC_WITH_IMM; 33500f7ec36c9324928e4cd23f02e6d8550f30c32caSteve Wise wc.ex.imm_data = wqe->wr.ex.imm_data; 336e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan /* FALLTHROUGH */ 337e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan case IB_WR_SEND: 33853dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell if (!ipath_get_rwqe(qp, 0)) 33953dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell goto rnr_nak; 340e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan break; 341e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 342e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan case IB_WR_RDMA_WRITE_WITH_IMM: 34353dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) 34453dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell goto inv_err; 345ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan wc.wc_flags = IB_WC_WITH_IMM; 34600f7ec36c9324928e4cd23f02e6d8550f30c32caSteve Wise wc.ex.imm_data = wqe->wr.ex.imm_data; 347e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (!ipath_get_rwqe(qp, 1)) 348e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto rnr_nak; 349e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan /* FALLTHROUGH */ 350e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan case IB_WR_RDMA_WRITE: 35153dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) 35253dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell goto inv_err; 353e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (wqe->length == 0) 354e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan break; 3556a553af286653818bb5831f1b351eefdc8a93b61Bryan O'Sullivan if (unlikely(!ipath_rkey_ok(qp, &qp->r_sge, wqe->length, 356e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->wr.wr.rdma.remote_addr, 357e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->wr.wr.rdma.rkey, 35853dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell IB_ACCESS_REMOTE_WRITE))) 35953dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell goto acc_err; 360e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan break; 361e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 362e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan case IB_WR_RDMA_READ: 36353dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) 36453dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell goto inv_err; 3656a553af286653818bb5831f1b351eefdc8a93b61Bryan O'Sullivan if (unlikely(!ipath_rkey_ok(qp, &sqp->s_sge, wqe->length, 366e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->wr.wr.rdma.remote_addr, 367e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->wr.wr.rdma.rkey, 368e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan IB_ACCESS_REMOTE_READ))) 369e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto acc_err; 370e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->r_sge.sge = wqe->sg_list[0]; 371e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->r_sge.sg_list = wqe->sg_list + 1; 372e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->r_sge.num_sge = wqe->wr.num_sge; 373e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan break; 374e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 375e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan case IB_WR_ATOMIC_CMP_AND_SWP: 376e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan case IB_WR_ATOMIC_FETCH_AND_ADD: 37753dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC))) 37853dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell goto inv_err; 3796a553af286653818bb5831f1b351eefdc8a93b61Bryan O'Sullivan if (unlikely(!ipath_rkey_ok(qp, &qp->r_sge, sizeof(u64), 3803859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.wr.atomic.remote_addr, 3813859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell wqe->wr.wr.atomic.rkey, 382e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan IB_ACCESS_REMOTE_ATOMIC))) 383e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto acc_err; 384e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan /* Perform atomic OP and save result. */ 3853859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell maddr = (atomic64_t *) qp->r_sge.sge.vaddr; 3863859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell sdata = wqe->wr.wr.atomic.compare_add; 3873859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell *(u64 *) sqp->s_sge.sge.vaddr = 3883859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell (wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) ? 3893859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell (u64) atomic64_add_return(sdata, maddr) - sdata : 3903859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr, 3913859e39d75b72f35f7d38c618fbbacb39a440c22Ralph Campbell sdata, wqe->wr.wr.atomic.swap); 392e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto send_comp; 393e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 394e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan default: 39553dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell send_status = IB_WC_LOC_QP_OP_ERR; 39653dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell goto serr; 397e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 398e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 399e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sge = &sqp->s_sge.sge; 400e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan while (sqp->s_len) { 401e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan u32 len = sqp->s_len; 402e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 403e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (len > sge->length) 404e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan len = sge->length; 40530d149ab58cc3ed8e4bc9c4dc45bebbed0e84b6eRalph Campbell if (len > sge->sge_length) 40630d149ab58cc3ed8e4bc9c4dc45bebbed0e84b6eRalph Campbell len = sge->sge_length; 407e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan BUG_ON(len == 0); 408e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ipath_copy_sge(&qp->r_sge, sge->vaddr, len); 409e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sge->vaddr += len; 410e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sge->length -= len; 411e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sge->sge_length -= len; 412e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (sge->sge_length == 0) { 413e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (--sqp->s_sge.num_sge) 414e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan *sge = *sqp->s_sge.sg_list++; 415e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } else if (sge->length == 0 && sge->mr != NULL) { 416e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (++sge->n >= IPATH_SEGSZ) { 417e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (++sge->m >= sge->mr->mapsz) 418e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan break; 419e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sge->n = 0; 420e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 421e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sge->vaddr = 422e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sge->mr->map[sge->m]->segs[sge->n].vaddr; 423e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sge->length = 424e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sge->mr->map[sge->m]->segs[sge->n].length; 425e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 426e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sqp->s_len -= len; 427e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 428e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 429e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (!test_and_clear_bit(IPATH_R_WRID_VALID, &qp->r_aflags)) 430e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto send_comp; 431e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 432e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM) 433ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan wc.opcode = IB_WC_RECV_RDMA_WITH_IMM; 434e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan else 435ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan wc.opcode = IB_WC_RECV; 436ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan wc.wr_id = qp->r_wr_id; 437ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan wc.status = IB_WC_SUCCESS; 438ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan wc.byte_len = wqe->length; 439062dbb69f32b9ccea701b30f8cc0049482e6211fMichael S. Tsirkin wc.qp = &qp->ibqp; 440ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan wc.src_qp = qp->remote_qpn; 441ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan wc.slid = qp->remote_ah_attr.dlid; 442ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan wc.sl = qp->remote_ah_attr.sl; 4434ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell wc.port_num = 1; 444e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan /* Signal completion event if the solicited bit is set. */ 445ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 446e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->wr.send_flags & IB_SEND_SOLICITED); 447e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 448e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivansend_comp: 449e509be898d8937634437caa474b57ac12795e5bcRalph Campbell spin_lock_irqsave(&sqp->s_lock, flags); 450e509be898d8937634437caa474b57ac12795e5bcRalph Campbellflush_send: 451e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sqp->s_rnr_retry = sqp->s_rnr_retry_cnt; 45253dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell ipath_send_complete(sqp, wqe, send_status); 453e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto again; 454e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 45553dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbellrnr_nak: 45653dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell /* Handle RNR NAK */ 45753dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell if (qp->ibqp.qp_type == IB_QPT_UC) 45853dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell goto send_comp; 45953dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell /* 46053dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell * Note: we don't need the s_lock held since the BUSY flag 46153dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell * makes this single threaded. 46253dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell */ 46353dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell if (sqp->s_rnr_retry == 0) { 46453dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell send_status = IB_WC_RNR_RETRY_EXC_ERR; 46553dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell goto serr; 46653dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell } 46753dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell if (sqp->s_rnr_retry_cnt < 7) 46853dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell sqp->s_rnr_retry--; 46953dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell spin_lock_irqsave(&sqp->s_lock, flags); 47053dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell if (!(ib_ipath_state_ops[sqp->state] & IPATH_PROCESS_RECV_OK)) 471e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto clr_busy; 472e509be898d8937634437caa474b57ac12795e5bcRalph Campbell sqp->s_flags |= IPATH_S_WAITING; 47353dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell dev->n_rnr_naks++; 47453dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell sqp->s_rnr_timeout = ib_ipath_rnr_table[qp->r_min_rnr_timer]; 47553dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell ipath_insert_rnr_queue(sqp); 476e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto clr_busy; 47753dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell 47853dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbellinv_err: 47953dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell send_status = IB_WC_REM_INV_REQ_ERR; 48053dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell wc.status = IB_WC_LOC_QP_OP_ERR; 48153dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell goto err; 48253dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell 48353dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbellacc_err: 48453dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell send_status = IB_WC_REM_ACCESS_ERR; 48553dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell wc.status = IB_WC_LOC_PROT_ERR; 48653dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbellerr: 48753dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell /* responder goes to error state */ 48853dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell ipath_rc_error(qp, wc.status); 48953dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell 49053dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbellserr: 49153dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell spin_lock_irqsave(&sqp->s_lock, flags); 49253dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell ipath_send_complete(sqp, wqe, send_status); 49353dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell if (sqp->ibqp.qp_type == IB_QPT_RC) { 49453dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell int lastwqe = ipath_error_qp(sqp, IB_WC_WR_FLUSH_ERR); 49553dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell 49653dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell sqp->s_flags &= ~IPATH_S_BUSY; 49753dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell spin_unlock_irqrestore(&sqp->s_lock, flags); 49853dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell if (lastwqe) { 49953dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell struct ib_event ev; 50053dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell 50153dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell ev.device = sqp->ibqp.device; 50253dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell ev.element.qp = &sqp->ibqp; 50353dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell ev.event = IB_EVENT_QP_LAST_WQE_REACHED; 50453dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell sqp->ibqp.event_handler(&ev, sqp->ibqp.qp_context); 50553dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell } 50653dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell goto done; 50753dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell } 508e509be898d8937634437caa474b57ac12795e5bcRalph Campbellclr_busy: 509e509be898d8937634437caa474b57ac12795e5bcRalph Campbell sqp->s_flags &= ~IPATH_S_BUSY; 51053dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbellunlock: 51153dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell spin_unlock_irqrestore(&sqp->s_lock, flags); 512e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivandone: 513e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (qp && atomic_dec_and_test(&qp->refcount)) 514e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wake_up(&qp->wait); 515e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan} 516e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 517e2ab41cae418108f376ad1634d7507f56379f7a2Dave Olsonstatic void want_buffer(struct ipath_devdata *dd, struct ipath_qp *qp) 51834b2aafea38efdf02cd8107a6e1057e2a297c447Bryan O'Sullivan{ 519e2ab41cae418108f376ad1634d7507f56379f7a2Dave Olson if (!(dd->ipath_flags & IPATH_HAS_SEND_DMA) || 520e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->ibqp.qp_type == IB_QPT_SMI) { 521124b4dcb1dd3a6fb80051f1785117a732d785f70Dave Olson unsigned long flags; 522124b4dcb1dd3a6fb80051f1785117a732d785f70Dave Olson 523124b4dcb1dd3a6fb80051f1785117a732d785f70Dave Olson spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags); 524124b4dcb1dd3a6fb80051f1785117a732d785f70Dave Olson dd->ipath_sendctrl |= INFINIPATH_S_PIOINTBUFAVAIL; 525124b4dcb1dd3a6fb80051f1785117a732d785f70Dave Olson ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, 526124b4dcb1dd3a6fb80051f1785117a732d785f70Dave Olson dd->ipath_sendctrl); 527124b4dcb1dd3a6fb80051f1785117a732d785f70Dave Olson ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch); 528124b4dcb1dd3a6fb80051f1785117a732d785f70Dave Olson spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags); 529124b4dcb1dd3a6fb80051f1785117a732d785f70Dave Olson } 53034b2aafea38efdf02cd8107a6e1057e2a297c447Bryan O'Sullivan} 53134b2aafea38efdf02cd8107a6e1057e2a297c447Bryan O'Sullivan 532e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan/** 533e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * ipath_no_bufs_available - tell the layer driver we need buffers 534e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * @qp: the QP that caused the problem 535e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * @dev: the device we ran out of buffers on 536e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * 537e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * Called when we run out of PIO buffers. 538e509be898d8937634437caa474b57ac12795e5bcRalph Campbell * If we are now in the error state, return zero to flush the 539e509be898d8937634437caa474b57ac12795e5bcRalph Campbell * send work request. 540e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan */ 541e509be898d8937634437caa474b57ac12795e5bcRalph Campbellstatic int ipath_no_bufs_available(struct ipath_qp *qp, 5424ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell struct ipath_ibdev *dev) 543e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan{ 544e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan unsigned long flags; 545e509be898d8937634437caa474b57ac12795e5bcRalph Campbell int ret = 1; 546e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 547e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan /* 54834b2aafea38efdf02cd8107a6e1057e2a297c447Bryan O'Sullivan * Note that as soon as want_buffer() is called and 549e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * possibly before it returns, ipath_ib_piobufavail() 550e509be898d8937634437caa474b57ac12795e5bcRalph Campbell * could be called. Therefore, put QP on the piowait list before 551e509be898d8937634437caa474b57ac12795e5bcRalph Campbell * enabling the PIO avail interrupt. 552e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan */ 553e509be898d8937634437caa474b57ac12795e5bcRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 554e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK) { 555e509be898d8937634437caa474b57ac12795e5bcRalph Campbell dev->n_piowait++; 556e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->s_flags |= IPATH_S_WAITING; 557e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->s_flags &= ~IPATH_S_BUSY; 558e509be898d8937634437caa474b57ac12795e5bcRalph Campbell spin_lock(&dev->pending_lock); 559e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (list_empty(&qp->piowait)) 560e509be898d8937634437caa474b57ac12795e5bcRalph Campbell list_add_tail(&qp->piowait, &dev->piowait); 561e509be898d8937634437caa474b57ac12795e5bcRalph Campbell spin_unlock(&dev->pending_lock); 562e509be898d8937634437caa474b57ac12795e5bcRalph Campbell } else 563e509be898d8937634437caa474b57ac12795e5bcRalph Campbell ret = 0; 564e509be898d8937634437caa474b57ac12795e5bcRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 565e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (ret) 566e509be898d8937634437caa474b57ac12795e5bcRalph Campbell want_buffer(dev->dd, qp); 567e509be898d8937634437caa474b57ac12795e5bcRalph Campbell return ret; 568e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan} 569e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 570e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan/** 571ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * ipath_make_grh - construct a GRH header 572ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * @dev: a pointer to the ipath device 573ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * @hdr: a pointer to the GRH header being constructed 574ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * @grh: the global route address to send to 575ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * @hwords: the number of 32 bit words of header being sent 576ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * @nwords: the number of 32 bit words of data being sent 577ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * 578ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * Return the size of the header in 32 bit words. 579ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan */ 580ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivanu32 ipath_make_grh(struct ipath_ibdev *dev, struct ib_grh *hdr, 581ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan struct ib_global_route *grh, u32 hwords, u32 nwords) 582ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan{ 583ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hdr->version_tclass_flow = 584ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan cpu_to_be32((6 << 28) | 585ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan (grh->traffic_class << 20) | 586ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan grh->flow_label); 587ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hdr->paylen = cpu_to_be16((hwords - 2 + nwords + SIZE_OF_CRC) << 2); 588ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan /* next_hdr is defined by C8-7 in ch. 8.4.1 */ 589ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hdr->next_hdr = 0x1B; 590ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hdr->hop_limit = grh->hop_limit; 591ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan /* The SGID is 32-bit aligned. */ 592ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hdr->sgid.global.subnet_prefix = dev->gid_prefix; 59334b2aafea38efdf02cd8107a6e1057e2a297c447Bryan O'Sullivan hdr->sgid.global.interface_id = dev->dd->ipath_guid; 594ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan hdr->dgid = grh->dgid; 595ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan 596ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan /* GRH header size in 32-bit words. */ 597ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan return sizeof(struct ib_grh) / sizeof(u32); 598ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan} 599ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan 6004ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbellvoid ipath_make_ruc_header(struct ipath_ibdev *dev, struct ipath_qp *qp, 6014ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell struct ipath_other_headers *ohdr, 6024ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell u32 bth0, u32 bth2) 6034ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell{ 6044ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell u16 lrh0; 6054ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell u32 nwords; 6064ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell u32 extra_bytes; 6074ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell 6084ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell /* Construct the header. */ 6094ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell extra_bytes = -qp->s_cur_size & 3; 6104ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell nwords = (qp->s_cur_size + extra_bytes) >> 2; 6114ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell lrh0 = IPATH_LRH_BTH; 6124ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) { 6134ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp->s_hdrwords += ipath_make_grh(dev, &qp->s_hdr.u.l.grh, 6144ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell &qp->remote_ah_attr.grh, 6154ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp->s_hdrwords, nwords); 6164ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell lrh0 = IPATH_LRH_GRH; 6174ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell } 6184ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell lrh0 |= qp->remote_ah_attr.sl << 4; 6194ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp->s_hdr.lrh[0] = cpu_to_be16(lrh0); 6204ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp->s_hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid); 6214ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp->s_hdr.lrh[2] = cpu_to_be16(qp->s_hdrwords + nwords + SIZE_OF_CRC); 622940358967599ba9057b3c51ba906e1cd5b984729Ralph Campbell qp->s_hdr.lrh[3] = cpu_to_be16(dev->dd->ipath_lid | 623940358967599ba9057b3c51ba906e1cd5b984729Ralph Campbell qp->remote_ah_attr.src_path_bits); 6244ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell bth0 |= ipath_get_pkey(dev->dd, qp->s_pkey_index); 6254ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell bth0 |= extra_bytes << 20; 6264ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell ohdr->bth[0] = cpu_to_be32(bth0 | (1 << 22)); 6274ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell ohdr->bth[1] = cpu_to_be32(qp->remote_qpn); 6284ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell ohdr->bth[2] = cpu_to_be32(bth2); 6294ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell} 6304ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell 631ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan/** 6324ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell * ipath_do_send - perform a send on a QP 633ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * @data: contains a pointer to the QP 634ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * 635ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * Process entries in the send work queue until credit or queue is 636ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * exhausted. Only allow one CPU to send a packet per QP (tasklet). 6374ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell * Otherwise, two threads could send packets out of order. 638ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan */ 6394ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbellvoid ipath_do_send(unsigned long data) 640ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan{ 641ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan struct ipath_qp *qp = (struct ipath_qp *)data; 642ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan struct ipath_ibdev *dev = to_idev(qp->ibqp.device); 6434ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell int (*make_req)(struct ipath_qp *qp); 644e509be898d8937634437caa474b57ac12795e5bcRalph Campbell unsigned long flags; 645ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan 6464ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell if ((qp->ibqp.qp_type == IB_QPT_RC || 6474ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp->ibqp.qp_type == IB_QPT_UC) && 6484ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp->remote_ah_attr.dlid == dev->dd->ipath_lid) { 649ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan ipath_ruc_loopback(qp); 650e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto bail; 651ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan } 652ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan 6534ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell if (qp->ibqp.qp_type == IB_QPT_RC) 6544ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell make_req = ipath_make_rc_req; 6554ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell else if (qp->ibqp.qp_type == IB_QPT_UC) 6564ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell make_req = ipath_make_uc_req; 6574ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell else 6584ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell make_req = ipath_make_ud_req; 659ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan 660e509be898d8937634437caa474b57ac12795e5bcRalph Campbell spin_lock_irqsave(&qp->s_lock, flags); 661e509be898d8937634437caa474b57ac12795e5bcRalph Campbell 662e509be898d8937634437caa474b57ac12795e5bcRalph Campbell /* Return if we are already busy processing a work request. */ 663e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if ((qp->s_flags & (IPATH_S_BUSY | IPATH_S_ANY_WAIT)) || 664e509be898d8937634437caa474b57ac12795e5bcRalph Campbell !(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_OR_FLUSH_SEND)) { 665e509be898d8937634437caa474b57ac12795e5bcRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 666e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto bail; 667e509be898d8937634437caa474b57ac12795e5bcRalph Campbell } 668e509be898d8937634437caa474b57ac12795e5bcRalph Campbell 669e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->s_flags |= IPATH_S_BUSY; 670e509be898d8937634437caa474b57ac12795e5bcRalph Campbell 671e509be898d8937634437caa474b57ac12795e5bcRalph Campbell spin_unlock_irqrestore(&qp->s_lock, flags); 672e509be898d8937634437caa474b57ac12795e5bcRalph Campbell 673ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivanagain: 674ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan /* Check for a constructed packet to be sent. */ 675ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan if (qp->s_hdrwords != 0) { 676ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan /* 677ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * If no PIO bufs are available, return. An interrupt will 678ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan * call ipath_ib_piobufavail() when one is available. 679ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan */ 6804ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell if (ipath_verbs_send(qp, &qp->s_hdr, qp->s_hdrwords, 6814ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell qp->s_cur_sge, qp->s_cur_size)) { 682e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (ipath_no_bufs_available(qp, dev)) 683e509be898d8937634437caa474b57ac12795e5bcRalph Campbell goto bail; 684ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan } 685ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan dev->n_unicast_xmit++; 686ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan /* Record that we sent the packet and s_hdr is empty. */ 687ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan qp->s_hdrwords = 0; 688ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan } 689ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan 6904ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell if (make_req(qp)) 6914ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell goto again; 692e509be898d8937634437caa474b57ac12795e5bcRalph Campbell 6934ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbellbail:; 6944ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell} 695ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan 696e509be898d8937634437caa474b57ac12795e5bcRalph Campbell/* 697e509be898d8937634437caa474b57ac12795e5bcRalph Campbell * This should be called with s_lock held. 698e509be898d8937634437caa474b57ac12795e5bcRalph Campbell */ 6994ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbellvoid ipath_send_complete(struct ipath_qp *qp, struct ipath_swqe *wqe, 7004ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell enum ib_wc_status status) 7014ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell{ 702e509be898d8937634437caa474b57ac12795e5bcRalph Campbell u32 old_last, last; 703e509be898d8937634437caa474b57ac12795e5bcRalph Campbell 704e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_OR_FLUSH_SEND)) 705e509be898d8937634437caa474b57ac12795e5bcRalph Campbell return; 706ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan 7074ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell /* See ch. 11.2.4.1 and 10.7.3.1 */ 7084ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell if (!(qp->s_flags & IPATH_S_SIGNAL_REQ_WR) || 7094ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell (wqe->wr.send_flags & IB_SEND_SIGNALED) || 7104ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell status != IB_WC_SUCCESS) { 7114ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell struct ib_wc wc; 712ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan 71353dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell memset(&wc, 0, sizeof wc); 7144ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell wc.wr_id = wqe->wr.wr_id; 7154ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell wc.status = status; 7164ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode]; 7174ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell wc.qp = &qp->ibqp; 71853dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell if (status == IB_WC_SUCCESS) 71953dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell wc.byte_len = wqe->length; 72053dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 72153dc1ca194c062aa9771e194047f27ec1ca592dfRalph Campbell status != IB_WC_SUCCESS); 7224ee97180ac76deb5a715ac45b7d7516e6ee82ae7Ralph Campbell } 723fffbfeaa680e2b87a591e141f2aa7e9e91184956Ralph Campbell 724e509be898d8937634437caa474b57ac12795e5bcRalph Campbell old_last = last = qp->s_last; 725fffbfeaa680e2b87a591e141f2aa7e9e91184956Ralph Campbell if (++last >= qp->s_size) 726fffbfeaa680e2b87a591e141f2aa7e9e91184956Ralph Campbell last = 0; 727fffbfeaa680e2b87a591e141f2aa7e9e91184956Ralph Campbell qp->s_last = last; 728e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (qp->s_cur == old_last) 729e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->s_cur = last; 730e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (qp->s_tail == old_last) 731e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->s_tail = last; 732e509be898d8937634437caa474b57ac12795e5bcRalph Campbell if (qp->state == IB_QPS_SQD && last == qp->s_cur) 733e509be898d8937634437caa474b57ac12795e5bcRalph Campbell qp->s_draining = 0; 734ddd4bb22108417fdc5c35324bd13a3265581ae76Bryan O'Sullivan} 735