ipath_ruc.c revision e28c00ad67164dba688c1d19c208c5fb554465f2
1e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan/* 2e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. 3e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * 4e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * This software is available to you under a choice of one of two 5e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * licenses. You may choose to be licensed under the terms of the GNU 6e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * General Public License (GPL) Version 2, available from the file 7e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * COPYING in the main directory of this source tree, or the 8e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * OpenIB.org BSD license below: 9e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * 10e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * Redistribution and use in source and binary forms, with or 11e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * without modification, are permitted provided that the following 12e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * conditions are met: 13e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * 14e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * - Redistributions of source code must retain the above 15e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * copyright notice, this list of conditions and the following 16e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * disclaimer. 17e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * 18e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * - Redistributions in binary form must reproduce the above 19e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * copyright notice, this list of conditions and the following 20e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * disclaimer in the documentation and/or other materials 21e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * provided with the distribution. 22e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * 23e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * SOFTWARE. 31e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan */ 32e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 33e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan#include "ipath_verbs.h" 34e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 35e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan/* 36e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * Convert the AETH RNR timeout code into the number of milliseconds. 37e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan */ 38e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivanconst u32 ib_ipath_rnr_table[32] = { 39e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 656, /* 0 */ 40e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* 1 */ 41e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* 2 */ 42e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* 3 */ 43e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* 4 */ 44e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* 5 */ 45e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* 6 */ 46e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* 7 */ 47e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* 8 */ 48e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* 9 */ 49e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* A */ 50e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* B */ 51e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* C */ 52e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 1, /* D */ 53e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 2, /* E */ 54e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 2, /* F */ 55e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 3, /* 10 */ 56e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 4, /* 11 */ 57e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 6, /* 12 */ 58e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 8, /* 13 */ 59e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 11, /* 14 */ 60e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 16, /* 15 */ 61e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 21, /* 16 */ 62e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 31, /* 17 */ 63e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 41, /* 18 */ 64e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 62, /* 19 */ 65e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 82, /* 1A */ 66e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 123, /* 1B */ 67e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 164, /* 1C */ 68e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 246, /* 1D */ 69e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 328, /* 1E */ 70e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 492 /* 1F */ 71e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan}; 72e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 73e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan/** 74e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * ipath_insert_rnr_queue - put QP on the RNR timeout list for the device 75e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * @qp: the QP 76e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * 77e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * XXX Use a simple list for now. We might need a priority 78e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * queue if we have lots of QPs waiting for RNR timeouts 79e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * but that should be rare. 80e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan */ 81e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivanvoid ipath_insert_rnr_queue(struct ipath_qp *qp) 82e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan{ 83e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct ipath_ibdev *dev = to_idev(qp->ibqp.device); 84e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan unsigned long flags; 85e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 86e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_lock_irqsave(&dev->pending_lock, flags); 87e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (list_empty(&dev->rnrwait)) 88e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan list_add(&qp->timerwait, &dev->rnrwait); 89e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan else { 90e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct list_head *l = &dev->rnrwait; 91e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct ipath_qp *nqp = list_entry(l->next, struct ipath_qp, 92e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan timerwait); 93e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 94e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan while (qp->s_rnr_timeout >= nqp->s_rnr_timeout) { 95e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->s_rnr_timeout -= nqp->s_rnr_timeout; 96e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan l = l->next; 97e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (l->next == &dev->rnrwait) 98e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan break; 99e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan nqp = list_entry(l->next, struct ipath_qp, 100e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan timerwait); 101e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 102e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan list_add(&qp->timerwait, l); 103e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 104e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_unlock_irqrestore(&dev->pending_lock, flags); 105e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan} 106e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 107e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan/** 108e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * ipath_get_rwqe - copy the next RWQE into the QP's RWQE 109e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * @qp: the QP 110e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * @wr_id_only: update wr_id only, not SGEs 111e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * 112e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * Return 0 if no RWQE is available, otherwise return 1. 113e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * 114e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * Called at interrupt level with the QP r_rq.lock held. 115e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan */ 116e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivanint ipath_get_rwqe(struct ipath_qp *qp, int wr_id_only) 117e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan{ 118e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct ipath_rq *rq; 119e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct ipath_srq *srq; 120e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct ipath_rwqe *wqe; 121e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan int ret; 122e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 123e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (!qp->ibqp.srq) { 124e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan rq = &qp->r_rq; 125e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (unlikely(rq->tail == rq->head)) { 126e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ret = 0; 127e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto bail; 128e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 129e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe = get_rwqe_ptr(rq, rq->tail); 130e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->r_wr_id = wqe->wr_id; 131e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (!wr_id_only) { 132e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->r_sge.sge = wqe->sg_list[0]; 133e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->r_sge.sg_list = wqe->sg_list + 1; 134e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->r_sge.num_sge = wqe->num_sge; 135e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->r_len = wqe->length; 136e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 137e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (++rq->tail >= rq->size) 138e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan rq->tail = 0; 139e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ret = 1; 140e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto bail; 141e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 142e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 143e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan srq = to_isrq(qp->ibqp.srq); 144e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan rq = &srq->rq; 145e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_lock(&rq->lock); 146e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (unlikely(rq->tail == rq->head)) { 147e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_unlock(&rq->lock); 148e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ret = 0; 149e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto bail; 150e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 151e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe = get_rwqe_ptr(rq, rq->tail); 152e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->r_wr_id = wqe->wr_id; 153e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (!wr_id_only) { 154e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->r_sge.sge = wqe->sg_list[0]; 155e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->r_sge.sg_list = wqe->sg_list + 1; 156e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->r_sge.num_sge = wqe->num_sge; 157e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->r_len = wqe->length; 158e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 159e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (++rq->tail >= rq->size) 160e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan rq->tail = 0; 161e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (srq->ibsrq.event_handler) { 162e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct ib_event ev; 163e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan u32 n; 164e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 165e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (rq->head < rq->tail) 166e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan n = rq->size + rq->head - rq->tail; 167e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan else 168e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan n = rq->head - rq->tail; 169e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (n < srq->limit) { 170e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan srq->limit = 0; 171e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_unlock(&rq->lock); 172e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ev.device = qp->ibqp.device; 173e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ev.element.srq = qp->ibqp.srq; 174e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ev.event = IB_EVENT_SRQ_LIMIT_REACHED; 175e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan srq->ibsrq.event_handler(&ev, 176e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan srq->ibsrq.srq_context); 177e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } else 178e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_unlock(&rq->lock); 179e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } else 180e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_unlock(&rq->lock); 181e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ret = 1; 182e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 183e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivanbail: 184e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan return ret; 185e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan} 186e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 187e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan/** 188e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * ipath_ruc_loopback - handle UC and RC lookback requests 189e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * @sqp: the loopback QP 190e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * @wc: the work completion entry 191e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * 192e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * This is called from ipath_do_uc_send() or ipath_do_rc_send() to 193e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * forward a WQE addressed to the same HCA. 194e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * Note that although we are single threaded due to the tasklet, we still 195e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * have to protect against post_send(). We don't have to worry about 196e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * receive interrupts since this is a connected protocol and all packets 197e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * will pass through here. 198e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan */ 199e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivanvoid ipath_ruc_loopback(struct ipath_qp *sqp, struct ib_wc *wc) 200e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan{ 201e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct ipath_ibdev *dev = to_idev(sqp->ibqp.device); 202e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct ipath_qp *qp; 203e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct ipath_swqe *wqe; 204e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct ipath_sge *sge; 205e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan unsigned long flags; 206e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan u64 sdata; 207e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 208e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp = ipath_lookup_qpn(&dev->qp_table, sqp->remote_qpn); 209e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (!qp) { 210e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan dev->n_pkt_drops++; 211e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan return; 212e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 213e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 214e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivanagain: 215e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_lock_irqsave(&sqp->s_lock, flags); 216e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 217e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (!(ib_ipath_state_ops[sqp->state] & IPATH_PROCESS_SEND_OK)) { 218e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_unlock_irqrestore(&sqp->s_lock, flags); 219e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto done; 220e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 221e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 222e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan /* Get the next send request. */ 223e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (sqp->s_last == sqp->s_head) { 224e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan /* Send work queue is empty. */ 225e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_unlock_irqrestore(&sqp->s_lock, flags); 226e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto done; 227e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 228e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 229e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan /* 230e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * We can rely on the entry not changing without the s_lock 231e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * being held until we update s_last. 232e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan */ 233e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe = get_swqe_ptr(sqp, sqp->s_last); 234e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_unlock_irqrestore(&sqp->s_lock, flags); 235e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 236e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->wc_flags = 0; 237e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->imm_data = 0; 238e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 239e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sqp->s_sge.sge = wqe->sg_list[0]; 240e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sqp->s_sge.sg_list = wqe->sg_list + 1; 241e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sqp->s_sge.num_sge = wqe->wr.num_sge; 242e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sqp->s_len = wqe->length; 243e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan switch (wqe->wr.opcode) { 244e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan case IB_WR_SEND_WITH_IMM: 245e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->wc_flags = IB_WC_WITH_IMM; 246e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->imm_data = wqe->wr.imm_data; 247e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan /* FALLTHROUGH */ 248e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan case IB_WR_SEND: 249e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_lock_irqsave(&qp->r_rq.lock, flags); 250e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (!ipath_get_rwqe(qp, 0)) { 251e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan rnr_nak: 252e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_unlock_irqrestore(&qp->r_rq.lock, flags); 253e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan /* Handle RNR NAK */ 254e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (qp->ibqp.qp_type == IB_QPT_UC) 255e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto send_comp; 256e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (sqp->s_rnr_retry == 0) { 257e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->status = IB_WC_RNR_RETRY_EXC_ERR; 258e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto err; 259e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 260e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (sqp->s_rnr_retry_cnt < 7) 261e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sqp->s_rnr_retry--; 262e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan dev->n_rnr_naks++; 263e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sqp->s_rnr_timeout = 264e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ib_ipath_rnr_table[sqp->s_min_rnr_timer]; 265e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ipath_insert_rnr_queue(sqp); 266e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto done; 267e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 268e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_unlock_irqrestore(&qp->r_rq.lock, flags); 269e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan break; 270e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 271e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan case IB_WR_RDMA_WRITE_WITH_IMM: 272e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->wc_flags = IB_WC_WITH_IMM; 273e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->imm_data = wqe->wr.imm_data; 274e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_lock_irqsave(&qp->r_rq.lock, flags); 275e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (!ipath_get_rwqe(qp, 1)) 276e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto rnr_nak; 277e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_unlock_irqrestore(&qp->r_rq.lock, flags); 278e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan /* FALLTHROUGH */ 279e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan case IB_WR_RDMA_WRITE: 280e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (wqe->length == 0) 281e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan break; 282e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (unlikely(!ipath_rkey_ok(dev, &qp->r_sge, wqe->length, 283e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->wr.wr.rdma.remote_addr, 284e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->wr.wr.rdma.rkey, 285e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan IB_ACCESS_REMOTE_WRITE))) { 286e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan acc_err: 287e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->status = IB_WC_REM_ACCESS_ERR; 288e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan err: 289e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->wr_id = wqe->wr.wr_id; 290e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->opcode = ib_ipath_wc_opcode[wqe->wr.opcode]; 291e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->vendor_err = 0; 292e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->byte_len = 0; 293e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->qp_num = sqp->ibqp.qp_num; 294e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->src_qp = sqp->remote_qpn; 295e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->pkey_index = 0; 296e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->slid = sqp->remote_ah_attr.dlid; 297e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->sl = sqp->remote_ah_attr.sl; 298e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->dlid_path_bits = 0; 299e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->port_num = 0; 300e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ipath_sqerror_qp(sqp, wc); 301e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto done; 302e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 303e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan break; 304e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 305e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan case IB_WR_RDMA_READ: 306e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (unlikely(!ipath_rkey_ok(dev, &sqp->s_sge, wqe->length, 307e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->wr.wr.rdma.remote_addr, 308e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->wr.wr.rdma.rkey, 309e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan IB_ACCESS_REMOTE_READ))) 310e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto acc_err; 311e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (unlikely(!(qp->qp_access_flags & 312e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan IB_ACCESS_REMOTE_READ))) 313e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto acc_err; 314e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->r_sge.sge = wqe->sg_list[0]; 315e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->r_sge.sg_list = wqe->sg_list + 1; 316e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->r_sge.num_sge = wqe->wr.num_sge; 317e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan break; 318e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 319e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan case IB_WR_ATOMIC_CMP_AND_SWP: 320e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan case IB_WR_ATOMIC_FETCH_AND_ADD: 321e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (unlikely(!ipath_rkey_ok(dev, &qp->r_sge, sizeof(u64), 322e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->wr.wr.rdma.remote_addr, 323e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->wr.wr.rdma.rkey, 324e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan IB_ACCESS_REMOTE_ATOMIC))) 325e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto acc_err; 326e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan /* Perform atomic OP and save result. */ 327e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sdata = wqe->wr.wr.atomic.swap; 328e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_lock_irqsave(&dev->pending_lock, flags); 329e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->r_atomic_data = *(u64 *) qp->r_sge.sge.vaddr; 330e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) 331e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan *(u64 *) qp->r_sge.sge.vaddr = 332e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->r_atomic_data + sdata; 333e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan else if (qp->r_atomic_data == wqe->wr.wr.atomic.compare_add) 334e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan *(u64 *) qp->r_sge.sge.vaddr = sdata; 335e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_unlock_irqrestore(&dev->pending_lock, flags); 336e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan *(u64 *) sqp->s_sge.sge.vaddr = qp->r_atomic_data; 337e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto send_comp; 338e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 339e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan default: 340e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto done; 341e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 342e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 343e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sge = &sqp->s_sge.sge; 344e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan while (sqp->s_len) { 345e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan u32 len = sqp->s_len; 346e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 347e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (len > sge->length) 348e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan len = sge->length; 349e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan BUG_ON(len == 0); 350e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ipath_copy_sge(&qp->r_sge, sge->vaddr, len); 351e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sge->vaddr += len; 352e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sge->length -= len; 353e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sge->sge_length -= len; 354e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (sge->sge_length == 0) { 355e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (--sqp->s_sge.num_sge) 356e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan *sge = *sqp->s_sge.sg_list++; 357e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } else if (sge->length == 0 && sge->mr != NULL) { 358e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (++sge->n >= IPATH_SEGSZ) { 359e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (++sge->m >= sge->mr->mapsz) 360e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan break; 361e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sge->n = 0; 362e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 363e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sge->vaddr = 364e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sge->mr->map[sge->m]->segs[sge->n].vaddr; 365e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sge->length = 366e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sge->mr->map[sge->m]->segs[sge->n].length; 367e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 368e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sqp->s_len -= len; 369e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 370e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 371e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_RDMA_WRITE || 372e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->wr.opcode == IB_WR_RDMA_READ) 373e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto send_comp; 374e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 375e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM) 376e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 377e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan else 378e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->opcode = IB_WC_RECV; 379e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->wr_id = qp->r_wr_id; 380e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->status = IB_WC_SUCCESS; 381e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->vendor_err = 0; 382e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->byte_len = wqe->length; 383e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->qp_num = qp->ibqp.qp_num; 384e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->src_qp = qp->remote_qpn; 385e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan /* XXX do we know which pkey matched? Only needed for GSI. */ 386e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->pkey_index = 0; 387e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->slid = qp->remote_ah_attr.dlid; 388e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->sl = qp->remote_ah_attr.sl; 389e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->dlid_path_bits = 0; 390e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan /* Signal completion event if the solicited bit is set. */ 391e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ipath_cq_enter(to_icq(qp->ibqp.recv_cq), wc, 392e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->wr.send_flags & IB_SEND_SOLICITED); 393e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 394e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivansend_comp: 395e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sqp->s_rnr_retry = sqp->s_rnr_retry_cnt; 396e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 397e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (!test_bit(IPATH_S_SIGNAL_REQ_WR, &sqp->s_flags) || 398e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan (wqe->wr.send_flags & IB_SEND_SIGNALED)) { 399e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->wr_id = wqe->wr.wr_id; 400e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->status = IB_WC_SUCCESS; 401e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->opcode = ib_ipath_wc_opcode[wqe->wr.opcode]; 402e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->vendor_err = 0; 403e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->byte_len = wqe->length; 404e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->qp_num = sqp->ibqp.qp_num; 405e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->src_qp = 0; 406e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->pkey_index = 0; 407e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->slid = 0; 408e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->sl = 0; 409e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->dlid_path_bits = 0; 410e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wc->port_num = 0; 411e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ipath_cq_enter(to_icq(sqp->ibqp.send_cq), wc, 0); 412e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 413e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 414e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan /* Update s_last now that we are finished with the SWQE */ 415e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_lock_irqsave(&sqp->s_lock, flags); 416e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (++sqp->s_last >= sqp->s_size) 417e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan sqp->s_last = 0; 418e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_unlock_irqrestore(&sqp->s_lock, flags); 419e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto again; 420e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 421e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivandone: 422e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (atomic_dec_and_test(&qp->refcount)) 423e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wake_up(&qp->wait); 424e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan} 425e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 426e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan/** 427e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * ipath_no_bufs_available - tell the layer driver we need buffers 428e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * @qp: the QP that caused the problem 429e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * @dev: the device we ran out of buffers on 430e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * 431e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * Called when we run out of PIO buffers. 432e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan */ 433e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivanvoid ipath_no_bufs_available(struct ipath_qp *qp, struct ipath_ibdev *dev) 434e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan{ 435e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan unsigned long flags; 436e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 437e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_lock_irqsave(&dev->pending_lock, flags); 438e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (qp->piowait.next == LIST_POISON1) 439e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan list_add_tail(&qp->piowait, &dev->piowait); 440e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_unlock_irqrestore(&dev->pending_lock, flags); 441e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan /* 442e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * Note that as soon as ipath_layer_want_buffer() is called and 443e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * possibly before it returns, ipath_ib_piobufavail() 444e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * could be called. If we are still in the tasklet function, 445e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * tasklet_hi_schedule() will not call us until the next time 446e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * tasklet_hi_schedule() is called. 447e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * We clear the tasklet flag now since we are committing to return 448e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * from the tasklet function. 449e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan */ 450e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan clear_bit(IPATH_S_BUSY, &qp->s_flags); 451e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan tasklet_unlock(&qp->s_task); 452e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ipath_layer_want_buffer(dev->dd); 453e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan dev->n_piowait++; 454e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan} 455e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 456e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan/** 457e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * ipath_post_rc_send - post RC and UC sends 458e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * @qp: the QP to post on 459e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * @wr: the work request to send 460e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan */ 461e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivanint ipath_post_rc_send(struct ipath_qp *qp, struct ib_send_wr *wr) 462e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan{ 463e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan struct ipath_swqe *wqe; 464e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan unsigned long flags; 465e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan u32 next; 466e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan int i, j; 467e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan int acc; 468e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan int ret; 469e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 470e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan /* 471e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * Don't allow RDMA reads or atomic operations on UC or 472e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * undefined operations. 473e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * Make sure buffer is large enough to hold the result for atomics. 474e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan */ 475e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (qp->ibqp.qp_type == IB_QPT_UC) { 476e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if ((unsigned) wr->opcode >= IB_WR_RDMA_READ) { 477e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ret = -EINVAL; 478e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto bail; 479e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 480e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } else if ((unsigned) wr->opcode > IB_WR_ATOMIC_FETCH_AND_ADD) { 481e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ret = -EINVAL; 482e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto bail; 483e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } else if (wr->opcode >= IB_WR_ATOMIC_CMP_AND_SWP && 484e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan (wr->num_sge == 0 || 485e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wr->sg_list[0].length < sizeof(u64) || 486e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wr->sg_list[0].addr & (sizeof(u64) - 1))) { 487e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ret = -EINVAL; 488e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto bail; 489e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 490e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan /* IB spec says that num_sge == 0 is OK. */ 491e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (wr->num_sge > qp->s_max_sge) { 492e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ret = -ENOMEM; 493e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto bail; 494e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 495e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_lock_irqsave(&qp->s_lock, flags); 496e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan next = qp->s_head + 1; 497e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (next >= qp->s_size) 498e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan next = 0; 499e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (next == qp->s_last) { 500e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_unlock_irqrestore(&qp->s_lock, flags); 501e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ret = -EINVAL; 502e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto bail; 503e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 504e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 505e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe = get_swqe_ptr(qp, qp->s_head); 506e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->wr = *wr; 507e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->ssn = qp->s_ssn++; 508e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->sg_list[0].mr = NULL; 509e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->sg_list[0].vaddr = NULL; 510e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->sg_list[0].length = 0; 511e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->sg_list[0].sge_length = 0; 512e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->length = 0; 513e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan acc = wr->opcode >= IB_WR_RDMA_READ ? IB_ACCESS_LOCAL_WRITE : 0; 514e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan for (i = 0, j = 0; i < wr->num_sge; i++) { 515e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (to_ipd(qp->ibqp.pd)->user && wr->sg_list[i].lkey == 0) { 516e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_unlock_irqrestore(&qp->s_lock, flags); 517e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ret = -EINVAL; 518e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto bail; 519e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 520e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (wr->sg_list[i].length == 0) 521e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan continue; 522e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (!ipath_lkey_ok(&to_idev(qp->ibqp.device)->lk_table, 523e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan &wqe->sg_list[j], &wr->sg_list[i], 524e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan acc)) { 525e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_unlock_irqrestore(&qp->s_lock, flags); 526e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ret = -EINVAL; 527e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan goto bail; 528e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 529e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->length += wr->sg_list[i].length; 530e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan j++; 531e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 532e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan wqe->wr.num_sge = j; 533e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan qp->s_head = next; 534e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan /* 535e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * Wake up the send tasklet if the QP is not waiting 536e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan * for an RNR timeout. 537e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan */ 538e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan next = qp->s_rnr_timeout; 539e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan spin_unlock_irqrestore(&qp->s_lock, flags); 540e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 541e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (next == 0) { 542e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan if (qp->ibqp.qp_type == IB_QPT_UC) 543e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ipath_do_uc_send((unsigned long) qp); 544e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan else 545e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ipath_do_rc_send((unsigned long) qp); 546e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan } 547e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 548e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan ret = 0; 549e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan 550e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivanbail: 551e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan return ret; 552e28c00ad67164dba688c1d19c208c5fb554465f2Bryan O'Sullivan} 553