verbs.c revision c93e986a295d537589efd0504f36ca952bd1a5be
1f58851e6b0f148fb4b2a1c6f70beb2f125863c0f\"Talpey, Thomas\/* 2c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved. 3c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 4c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * This software is available to you under a choice of one of two 5c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * licenses. You may choose to be licensed under the terms of the GNU 6c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * General Public License (GPL) Version 2, available from the file 7c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * COPYING in the main directory of this source tree, or the BSD-type 8c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * license below: 9c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 10c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Redistribution and use in source and binary forms, with or without 11c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * modification, are permitted provided that the following conditions 12c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * are met: 13c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 14c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Redistributions of source code must retain the above copyright 15c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * notice, this list of conditions and the following disclaimer. 16c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 17c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Redistributions in binary form must reproduce the above 18c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * copyright notice, this list of conditions and the following 19c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * disclaimer in the documentation and/or other materials provided 20c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * with the distribution. 21c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 22c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Neither the name of the Network Appliance, Inc. nor the names of 23c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * its contributors may be used to endorse or promote products 24c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * derived from this software without specific prior written 25c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * permission. 26c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 27c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 28c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 29c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 30c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 31c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 32c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 33c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 34c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 35c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 36c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 37c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 38f58851e6b0f148fb4b2a1c6f70beb2f125863c0f\"Talpey, Thomas\ */ 39f58851e6b0f148fb4b2a1c6f70beb2f125863c0f\"Talpey, Thomas\ 40c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 41c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * verbs.c 42c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 43c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Encapsulates the major functions managing: 44c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o adapters 45c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o endpoints 46c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o connections 47c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o buffer memory 48c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 49c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 50a6b7a407865aab9f849dd99a71072b7cd1175116Alexey Dobriyan#include <linux/interrupt.h> 515a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 5265866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever#include <asm/bitops.h> 53c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 54f58851e6b0f148fb4b2a1c6f70beb2f125863c0f\"Talpey, Thomas\#include "xprt_rdma.h" 55f58851e6b0f148fb4b2a1c6f70beb2f125863c0f\"Talpey, Thomas\ 56c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 57c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Globals/Macros 58c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 59c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 60c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#ifdef RPC_DEBUG 61c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\# define RPCDBG_FACILITY RPCDBG_TRANS 62c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#endif 63c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 64c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 65c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * internal functions 66c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 67c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 68c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 69c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * handle replies in tasklet context, using a single, global list 70c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * rdma tasklet function -- just turn around and call the func 71c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * for all replies on the list 72c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 73c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 74c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static DEFINE_SPINLOCK(rpcrdma_tk_lock_g); 75c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static LIST_HEAD(rpcrdma_tasklets_g); 76c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 77c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 78c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_run_tasklet(unsigned long data) 79c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 80c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep *rep; 81c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ void (*func)(struct rpcrdma_rep *); 82c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 83c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 84c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ data = data; 85c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&rpcrdma_tk_lock_g, flags); 86c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ while (!list_empty(&rpcrdma_tasklets_g)) { 87c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep = list_entry(rpcrdma_tasklets_g.next, 88c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep, rr_list); 89c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_del(&rep->rr_list); 90c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ func = rep->rr_func; 91c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_func = NULL; 92c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&rpcrdma_tk_lock_g, flags); 93c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 94c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (func) 95c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ func(rep); 96c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ else 97c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_recv_buffer_put(rep); 98c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 99c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&rpcrdma_tk_lock_g, flags); 100c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 101c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&rpcrdma_tk_lock_g, flags); 102c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 103c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 104c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static DECLARE_TASKLET(rpcrdma_tasklet_g, rpcrdma_run_tasklet, 0UL); 105c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 106c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static inline void 107c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_schedule_tasklet(struct rpcrdma_rep *rep) 108c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 109c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 110c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 111c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&rpcrdma_tk_lock_g, flags); 112c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_add_tail(&rep->rr_list, &rpcrdma_tasklets_g); 113c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&rpcrdma_tk_lock_g, flags); 114c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ tasklet_schedule(&rpcrdma_tasklet_g); 115c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 116c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 117c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 118c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_qp_async_error_upcall(struct ib_event *event, void *context) 119c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 120c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ep *ep = context; 121c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 122c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: QP error %X on device %s ep %p\n", 123c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, event->event, event->device->name, context); 124c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_connected == 1) { 125c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = -EIO; 126c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_func(ep); 127c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wake_up_all(&ep->rep_connect_wait); 128c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 129c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 130c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 131c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 132c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_cq_async_error_upcall(struct ib_event *event, void *context) 133c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 134c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ep *ep = context; 135c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 136c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: CQ error %X on device %s ep %p\n", 137c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, event->event, event->device->name, context); 138c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_connected == 1) { 139c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = -EIO; 140c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_func(ep); 141c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wake_up_all(&ep->rep_connect_wait); 142c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 143c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 144c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 145fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Leverstatic void 146fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Leverrpcrdma_sendcq_process_wc(struct ib_wc *wc) 147c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 148fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever struct rpcrdma_mw *frmr = (struct rpcrdma_mw *)(unsigned long)wc->wr_id; 149c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 150fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever dprintk("RPC: %s: frmr %p status %X opcode %d\n", 151fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever __func__, frmr, wc->status, wc->opcode); 152c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 153fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (wc->wr_id == 0ULL) 154c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return; 155fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (wc->status != IB_WC_SUCCESS) 156c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return; 157c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 158fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (wc->opcode == IB_WC_FAST_REG_MR) 1590dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever frmr->r.frmr.fr_state = FRMR_IS_VALID; 160fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever else if (wc->opcode == IB_WC_LOCAL_INV) 1610dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever frmr->r.frmr.fr_state = FRMR_IS_INVALID; 162c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 163c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 164fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Leverstatic int 1651c00dd0776543608e13c74a527660cb8cd28a74fChuck Leverrpcrdma_sendcq_poll(struct ib_cq *cq, struct rpcrdma_ep *ep) 166c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1671c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever struct ib_wc *wcs; 1688301a2c047cc25dabd645e5590c1db0ead4c5af4Chuck Lever int budget, count, rc; 169c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1708301a2c047cc25dabd645e5590c1db0ead4c5af4Chuck Lever budget = RPCRDMA_WC_BUDGET / RPCRDMA_POLLSIZE; 1711c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever do { 1721c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever wcs = ep->rep_send_wcs; 1731c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever 1741c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever rc = ib_poll_cq(cq, RPCRDMA_POLLSIZE, wcs); 1751c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever if (rc <= 0) 1761c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever return rc; 1771c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever 1781c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever count = rc; 1791c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever while (count-- > 0) 1801c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever rpcrdma_sendcq_process_wc(wcs++); 1818301a2c047cc25dabd645e5590c1db0ead4c5af4Chuck Lever } while (rc == RPCRDMA_POLLSIZE && --budget); 1821c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever return 0; 183fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever} 184c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 185fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever/* 186fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever * Handle send, fast_reg_mr, and local_inv completions. 187fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever * 188fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever * Send events are typically suppressed and thus do not result 189fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever * in an upcall. Occasionally one is signaled, however. This 190fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever * prevents the provider's completion queue from wrapping and 191fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever * losing a completion. 192fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever */ 193fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Leverstatic void 194fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Leverrpcrdma_sendcq_upcall(struct ib_cq *cq, void *cq_context) 195fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever{ 1961c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever struct rpcrdma_ep *ep = (struct rpcrdma_ep *)cq_context; 197fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever int rc; 198fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 1991c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever rc = rpcrdma_sendcq_poll(cq, ep); 200fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (rc) { 201fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever dprintk("RPC: %s: ib_poll_cq failed: %i\n", 202fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever __func__, rc); 203fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever return; 204c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 205c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 2067f23f6f6e388d2003c4ecf5d558f3c2191e12530Chuck Lever rc = ib_req_notify_cq(cq, 2077f23f6f6e388d2003c4ecf5d558f3c2191e12530Chuck Lever IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS); 2087f23f6f6e388d2003c4ecf5d558f3c2191e12530Chuck Lever if (rc == 0) 2097f23f6f6e388d2003c4ecf5d558f3c2191e12530Chuck Lever return; 2107f23f6f6e388d2003c4ecf5d558f3c2191e12530Chuck Lever if (rc < 0) { 211fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever dprintk("RPC: %s: ib_req_notify_cq failed: %i\n", 212fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever __func__, rc); 213fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever return; 214fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever } 215fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 2161c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever rpcrdma_sendcq_poll(cq, ep); 217fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever} 218fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 219fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Leverstatic void 220fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Leverrpcrdma_recvcq_process_wc(struct ib_wc *wc) 221fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever{ 222fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever struct rpcrdma_rep *rep = 223fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever (struct rpcrdma_rep *)(unsigned long)wc->wr_id; 224fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 225fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever dprintk("RPC: %s: rep %p status %X opcode %X length %u\n", 226fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever __func__, rep, wc->status, wc->opcode, wc->byte_len); 227fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 228fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (wc->status != IB_WC_SUCCESS) { 229fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rep->rr_len = ~0U; 230fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever goto out_schedule; 231fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever } 232fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (wc->opcode != IB_WC_RECV) 233fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever return; 234fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 235fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rep->rr_len = wc->byte_len; 236fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever ib_dma_sync_single_for_cpu(rdmab_to_ia(rep->rr_buffer)->ri_id->device, 237fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rep->rr_iov.addr, rep->rr_len, DMA_FROM_DEVICE); 238fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 239fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (rep->rr_len >= 16) { 240fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever struct rpcrdma_msg *p = (struct rpcrdma_msg *)rep->rr_base; 241fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever unsigned int credits = ntohl(p->rm_credit); 242fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 243fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (credits == 0) 244fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever credits = 1; /* don't deadlock */ 245fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever else if (credits > rep->rr_buffer->rb_max_requests) 246fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever credits = rep->rr_buffer->rb_max_requests; 247fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever atomic_set(&rep->rr_buffer->rb_credits, credits); 248fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever } 249fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 250fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Leverout_schedule: 251fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rpcrdma_schedule_tasklet(rep); 252fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever} 253fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 254fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Leverstatic int 2551c00dd0776543608e13c74a527660cb8cd28a74fChuck Leverrpcrdma_recvcq_poll(struct ib_cq *cq, struct rpcrdma_ep *ep) 256fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever{ 2571c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever struct ib_wc *wcs; 2588301a2c047cc25dabd645e5590c1db0ead4c5af4Chuck Lever int budget, count, rc; 259fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 2608301a2c047cc25dabd645e5590c1db0ead4c5af4Chuck Lever budget = RPCRDMA_WC_BUDGET / RPCRDMA_POLLSIZE; 2611c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever do { 2621c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever wcs = ep->rep_recv_wcs; 2631c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever 2641c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever rc = ib_poll_cq(cq, RPCRDMA_POLLSIZE, wcs); 2651c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever if (rc <= 0) 2661c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever return rc; 2671c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever 2681c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever count = rc; 2691c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever while (count-- > 0) 2701c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever rpcrdma_recvcq_process_wc(wcs++); 2718301a2c047cc25dabd645e5590c1db0ead4c5af4Chuck Lever } while (rc == RPCRDMA_POLLSIZE && --budget); 2721c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever return 0; 273c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 274c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 275c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 276fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever * Handle receive completions. 277c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 278c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * It is reentrant but processes single events in order to maintain 279c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * ordering of receives to keep server credits. 280c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 281c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * It is the responsibility of the scheduled tasklet to return 282c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * recv buffers to the pool. NOTE: this affects synchronization of 283c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * connection shutdown. That is, the structures required for 284c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * the completion of the reply handler must remain intact until 285c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * all memory has been reclaimed. 286c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 287c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 288fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Leverrpcrdma_recvcq_upcall(struct ib_cq *cq, void *cq_context) 289c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 2901c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever struct rpcrdma_ep *ep = (struct rpcrdma_ep *)cq_context; 291c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 292c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 2931c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever rc = rpcrdma_recvcq_poll(cq, ep); 294fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (rc) { 295fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever dprintk("RPC: %s: ib_poll_cq failed: %i\n", 296fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever __func__, rc); 297c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return; 298fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever } 299c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 3007f23f6f6e388d2003c4ecf5d558f3c2191e12530Chuck Lever rc = ib_req_notify_cq(cq, 3017f23f6f6e388d2003c4ecf5d558f3c2191e12530Chuck Lever IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS); 3027f23f6f6e388d2003c4ecf5d558f3c2191e12530Chuck Lever if (rc == 0) 3037f23f6f6e388d2003c4ecf5d558f3c2191e12530Chuck Lever return; 3047f23f6f6e388d2003c4ecf5d558f3c2191e12530Chuck Lever if (rc < 0) { 305fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever dprintk("RPC: %s: ib_req_notify_cq failed: %i\n", 306c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 307c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return; 308c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 309c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 3101c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever rpcrdma_recvcq_poll(cq, ep); 311c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 312c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 313a7bc211ac926172ad20463afcf00ae7b9ebcd950Chuck Leverstatic void 314a7bc211ac926172ad20463afcf00ae7b9ebcd950Chuck Leverrpcrdma_flush_cqs(struct rpcrdma_ep *ep) 315a7bc211ac926172ad20463afcf00ae7b9ebcd950Chuck Lever{ 316a7bc211ac926172ad20463afcf00ae7b9ebcd950Chuck Lever rpcrdma_recvcq_upcall(ep->rep_attr.recv_cq, ep); 317a7bc211ac926172ad20463afcf00ae7b9ebcd950Chuck Lever rpcrdma_sendcq_upcall(ep->rep_attr.send_cq, ep); 318a7bc211ac926172ad20463afcf00ae7b9ebcd950Chuck Lever} 319a7bc211ac926172ad20463afcf00ae7b9ebcd950Chuck Lever 320c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#ifdef RPC_DEBUG 321c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static const char * const conn[] = { 322c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "address resolved", 323c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "address error", 324c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "route resolved", 325c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "route error", 326c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "connect request", 327c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "connect response", 328c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "connect error", 329c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "unreachable", 330c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "rejected", 331c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "established", 332c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "disconnected", 333c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "device removal" 334c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\}; 335c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#endif 336c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 337c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static int 338c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_conn_upcall(struct rdma_cm_id *id, struct rdma_cm_event *event) 339c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 340c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_xprt *xprt = id->context; 341c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = &xprt->rx_ia; 342c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ep *ep = &xprt->rx_ep; 343ff0db0490a2eb6db4f03870987c5c8e3dd81e747Ingo Molnar#ifdef RPC_DEBUG 344c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct sockaddr_in *addr = (struct sockaddr_in *) &ep->rep_remote_addr; 345ff0db0490a2eb6db4f03870987c5c8e3dd81e747Ingo Molnar#endif 346c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_qp_attr attr; 347c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_qp_init_attr iattr; 348c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int connstate = 0; 349c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 350c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (event->event) { 351c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_ADDR_RESOLVED: 352c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_ROUTE_RESOLVED: 3535675add36e76b9487e7f9e689f854cb8d6afd9b4Tom Talpey ia->ri_async_rc = 0; 354c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ complete(&ia->ri_done); 355c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 356c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_ADDR_ERROR: 357c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_async_rc = -EHOSTUNREACH; 358c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: CM address resolution error, ep 0x%p\n", 359c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, ep); 360c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ complete(&ia->ri_done); 361c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 362c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_ROUTE_ERROR: 363c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_async_rc = -ENETUNREACH; 364c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: CM route resolution error, ep 0x%p\n", 365c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, ep); 366c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ complete(&ia->ri_done); 367c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 368c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_ESTABLISHED: 369c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = 1; 370c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_query_qp(ia->ri_id->qp, &attr, 371c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ IB_QP_MAX_QP_RD_ATOMIC | IB_QP_MAX_DEST_RD_ATOMIC, 372c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &iattr); 373c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: %d responder resources" 374c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " (%d initiator)\n", 375c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, attr.max_dest_rd_atomic, attr.max_rd_atomic); 376c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto connected; 377c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_CONNECT_ERROR: 378c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = -ENOTCONN; 379c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto connected; 380c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_UNREACHABLE: 381c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = -ENETDOWN; 382c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto connected; 383c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_REJECTED: 384c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = -ECONNREFUSED; 385c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto connected; 386c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_DISCONNECTED: 387c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = -ECONNABORTED; 388c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto connected; 389c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_DEVICE_REMOVAL: 390c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = -ENODEV; 391c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\connected: 39221454aaad30651ba0dcc16fe5271bc12ee21f132Harvey Harrison dprintk("RPC: %s: %s: %pI4:%u (ep 0x%p event 0x%x)\n", 393c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, 394c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (event->event <= 11) ? conn[event->event] : 395c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "unknown connection error", 39621454aaad30651ba0dcc16fe5271bc12ee21f132Harvey Harrison &addr->sin_addr.s_addr, 397c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ntohs(addr->sin_port), 398c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep, event->event); 399c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ atomic_set(&rpcx_to_rdmax(ep->rep_xprt)->rx_buf.rb_credits, 1); 400c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: %sconnected\n", 401c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, connstate > 0 ? "" : "dis"); 402c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = connstate; 403c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_func(ep); 404c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wake_up_all(&ep->rep_connect_wait); 405c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 406c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 4071a954051b0cf79bd67e5f9db40333e3a9b1d05d2Tom Talpey dprintk("RPC: %s: unexpected CM event %d\n", 408c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, event->event); 409c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 410c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 411c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 412b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey#ifdef RPC_DEBUG 413b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey if (connstate == 1) { 414b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey int ird = attr.max_dest_rd_atomic; 415b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey int tird = ep->rep_remote_cma.responder_resources; 41621454aaad30651ba0dcc16fe5271bc12ee21f132Harvey Harrison printk(KERN_INFO "rpcrdma: connection to %pI4:%u " 417b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey "on %s, memreg %d slots %d ird %d%s\n", 41821454aaad30651ba0dcc16fe5271bc12ee21f132Harvey Harrison &addr->sin_addr.s_addr, 419b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey ntohs(addr->sin_port), 420b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey ia->ri_id->device->name, 421b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey ia->ri_memreg_strategy, 422b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey xprt->rx_buf.rb_max_requests, 423b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey ird, ird < 4 && ird < tird / 2 ? " (low!)" : ""); 424b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey } else if (connstate < 0) { 42521454aaad30651ba0dcc16fe5271bc12ee21f132Harvey Harrison printk(KERN_INFO "rpcrdma: connection to %pI4:%u closed (%d)\n", 42621454aaad30651ba0dcc16fe5271bc12ee21f132Harvey Harrison &addr->sin_addr.s_addr, 427b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey ntohs(addr->sin_port), 428b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey connstate); 429b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey } 430b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey#endif 431b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey 432c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 433c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 434c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 435c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static struct rdma_cm_id * 436c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_create_id(struct rpcrdma_xprt *xprt, 437c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia, struct sockaddr *addr) 438c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 439c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rdma_cm_id *id; 440c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 441c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 4421a954051b0cf79bd67e5f9db40333e3a9b1d05d2Tom Talpey init_completion(&ia->ri_done); 4431a954051b0cf79bd67e5f9db40333e3a9b1d05d2Tom Talpey 444b26f9b9949013fec31b23c426fc463164ae08891Sean Hefty id = rdma_create_id(rpcrdma_conn_upcall, xprt, RDMA_PS_TCP, IB_QPT_RC); 445c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(id)) { 446c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(id); 447c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_create_id() failed %i\n", 448c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 449c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return id; 450c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 451c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 4525675add36e76b9487e7f9e689f854cb8d6afd9b4Tom Talpey ia->ri_async_rc = -ETIMEDOUT; 453c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rdma_resolve_addr(id, NULL, addr, RDMA_RESOLVE_TIMEOUT); 454c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 455c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_resolve_addr() failed %i\n", 456c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 457c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 458c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 4595675add36e76b9487e7f9e689f854cb8d6afd9b4Tom Talpey wait_for_completion_interruptible_timeout(&ia->ri_done, 4605675add36e76b9487e7f9e689f854cb8d6afd9b4Tom Talpey msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1); 461c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ia->ri_async_rc; 462c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 463c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 464c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 4655675add36e76b9487e7f9e689f854cb8d6afd9b4Tom Talpey ia->ri_async_rc = -ETIMEDOUT; 466c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rdma_resolve_route(id, RDMA_RESOLVE_TIMEOUT); 467c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 468c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_resolve_route() failed %i\n", 469c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 470c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 471c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 4725675add36e76b9487e7f9e689f854cb8d6afd9b4Tom Talpey wait_for_completion_interruptible_timeout(&ia->ri_done, 4735675add36e76b9487e7f9e689f854cb8d6afd9b4Tom Talpey msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1); 474c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ia->ri_async_rc; 475c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 476c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 477c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 478c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return id; 479c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 480c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out: 481c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rdma_destroy_id(id); 482c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return ERR_PTR(rc); 483c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 484c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 485c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 486c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Drain any cq, prior to teardown. 487c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 488c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 489c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_clean_cq(struct ib_cq *cq) 490c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 491c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_wc wc; 492c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int count = 0; 493c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 494c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ while (1 == ib_poll_cq(cq, 1, &wc)) 495c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ++count; 496c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 497c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (count) 498c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: flushed %d events (last 0x%x)\n", 499c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, count, wc.opcode); 500c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 501c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 502c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 503c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Exported functions. 504c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 505c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 506c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 507c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Open and initialize an Interface Adapter. 508c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o initializes fields of struct rpcrdma_ia, including 509c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * interface and provider attributes and protection zone. 510c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 511c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 512c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ia_open(struct rpcrdma_xprt *xprt, struct sockaddr *addr, int memreg) 513c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 514bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey int rc, mem_priv; 515bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey struct ib_device_attr devattr; 516c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = &xprt->rx_ia; 517c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 518c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_id = rpcrdma_create_id(xprt, ia, addr); 519c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(ia->ri_id)) { 520c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(ia->ri_id); 521c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out1; 522c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 523c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 524c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_pd = ib_alloc_pd(ia->ri_id->device); 525c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(ia->ri_pd)) { 526c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(ia->ri_pd); 527c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_alloc_pd() failed %i\n", 528c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 529c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out2; 530c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 531c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 532c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 533bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey * Query the device to determine if the requested memory 534bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey * registration strategy is supported. If it isn't, set the 535bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey * strategy to a globally supported model. 536bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey */ 537bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey rc = ib_query_device(ia->ri_id->device, &devattr); 538bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey if (rc) { 539bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey dprintk("RPC: %s: ib_query_device failed %d\n", 540bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey __func__, rc); 541bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey goto out2; 542bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey } 543bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey 544bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey if (devattr.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY) { 545bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey ia->ri_have_dma_lkey = 1; 546bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey ia->ri_dma_lkey = ia->ri_id->device->local_dma_lkey; 547bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey } 548bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey 549f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever if (memreg == RPCRDMA_FRMR) { 5503197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey /* Requires both frmr reg and local dma lkey */ 5513197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey if ((devattr.device_cap_flags & 5523197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey (IB_DEVICE_MEM_MGT_EXTENSIONS|IB_DEVICE_LOCAL_DMA_LKEY)) != 5533197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey (IB_DEVICE_MEM_MGT_EXTENSIONS|IB_DEVICE_LOCAL_DMA_LKEY)) { 5543197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey dprintk("RPC: %s: FRMR registration " 555f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever "not supported by HCA\n", __func__); 556f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever memreg = RPCRDMA_MTHCAFMR; 5570fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise } else { 5580fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise /* Mind the ia limit on FRMR page list depth */ 5590fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise ia->ri_max_frmr_depth = min_t(unsigned int, 5600fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise RPCRDMA_MAX_DATA_SEGS, 5610fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise devattr.max_fast_reg_page_list_len); 5623197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey } 563f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever } 564f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever if (memreg == RPCRDMA_MTHCAFMR) { 565f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever if (!ia->ri_id->device->alloc_fmr) { 566f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever dprintk("RPC: %s: MTHCAFMR registration " 567f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever "not supported by HCA\n", __func__); 568f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever#if RPCRDMA_PERSISTENT_REGISTRATION 569f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever memreg = RPCRDMA_ALLPHYSICAL; 570f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever#else 571cdd9ade711599e7672a635add0406080856f8b92Chuck Lever rc = -ENOMEM; 572f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever goto out2; 573f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever#endif 574f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever } 575bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey } 576bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey 577bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey /* 578c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Optionally obtain an underlying physical identity mapping in 579c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * order to do a memory window-based bind. This base registration 580c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * is protected from remote access - that is enabled only by binding 581c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * for the specific bytes targeted during each RPC operation, and 582c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * revoked after the corresponding completion similar to a storage 583c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * adapter. 584c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 585bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey switch (memreg) { 5863197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey case RPCRDMA_FRMR: 587bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey break; 588c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#if RPCRDMA_PERSISTENT_REGISTRATION 589bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_ALLPHYSICAL: 590bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey mem_priv = IB_ACCESS_LOCAL_WRITE | 591bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey IB_ACCESS_REMOTE_WRITE | 592bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey IB_ACCESS_REMOTE_READ; 593bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey goto register_setup; 594c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#endif 595bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_MTHCAFMR: 596bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey if (ia->ri_have_dma_lkey) 597c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 598bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey mem_priv = IB_ACCESS_LOCAL_WRITE; 599b45ccfd25d506e83d9ecf93d0ac7edf031d35d2fChuck Lever#if RPCRDMA_PERSISTENT_REGISTRATION 600bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey register_setup: 601b45ccfd25d506e83d9ecf93d0ac7edf031d35d2fChuck Lever#endif 602c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_bind_mem = ib_get_dma_mr(ia->ri_pd, mem_priv); 603c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(ia->ri_bind_mem)) { 604c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ printk(KERN_ALERT "%s: ib_get_dma_mr for " 6050ac531c1832318efa3dc3d723e356a7e09330e80Chuck Lever "phys register failed with %lX\n", 606c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, PTR_ERR(ia->ri_bind_mem)); 6070ac531c1832318efa3dc3d723e356a7e09330e80Chuck Lever rc = -ENOMEM; 6080ac531c1832318efa3dc3d723e356a7e09330e80Chuck Lever goto out2; 609c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 610bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey break; 611bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey default: 612cdd9ade711599e7672a635add0406080856f8b92Chuck Lever printk(KERN_ERR "RPC: Unsupported memory " 613cdd9ade711599e7672a635add0406080856f8b92Chuck Lever "registration mode: %d\n", memreg); 614cdd9ade711599e7672a635add0406080856f8b92Chuck Lever rc = -ENOMEM; 615bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey goto out2; 616c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 617bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey dprintk("RPC: %s: memory registration strategy is %d\n", 618bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey __func__, memreg); 619c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 620c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Else will do memory reg/dereg for each chunk */ 621c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_memreg_strategy = memreg; 622c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 62373806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever rwlock_init(&ia->ri_qplock); 624c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 625c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out2: 626c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rdma_destroy_id(ia->ri_id); 627fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey ia->ri_id = NULL; 628c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out1: 629c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 630c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 631c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 632c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 633c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Clean up/close an IA. 634c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o if event handles and PD have been initialized, free them. 635c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o close the IA 636c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 637c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void 638c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ia_close(struct rpcrdma_ia *ia) 639c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 640c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 641c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 642c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: entering\n", __func__); 643c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ia->ri_bind_mem != NULL) { 644c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_dereg_mr(ia->ri_bind_mem); 645c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_dereg_mr returned %i\n", 646c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 647c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 648fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey if (ia->ri_id != NULL && !IS_ERR(ia->ri_id)) { 649fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey if (ia->ri_id->qp) 650fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey rdma_destroy_qp(ia->ri_id); 651fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey rdma_destroy_id(ia->ri_id); 652fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey ia->ri_id = NULL; 653fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey } 654c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ia->ri_pd != NULL && !IS_ERR(ia->ri_pd)) { 655c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_dealloc_pd(ia->ri_pd); 656c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_dealloc_pd returned %i\n", 657c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 658c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 659c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 660c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 661c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 662c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Create unconnected endpoint. 663c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 664c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 665c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia, 666c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_create_data_internal *cdata) 667c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 668c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_device_attr devattr; 669fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever struct ib_cq *sendcq, *recvcq; 6705d40a8a525c8165bafed233cf0f137e8d10d7e92Chuck Lever int rc, err; 671c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 672c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_query_device(ia->ri_id->device, &devattr); 673c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 674c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_query_device failed %d\n", 675c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 676c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 677c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 678c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 679c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* check provider's send/recv wr limits */ 680c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (cdata->max_requests > devattr.max_qp_wr) 681c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ cdata->max_requests = devattr.max_qp_wr; 682c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 683c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.event_handler = rpcrdma_qp_async_error_upcall; 684c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.qp_context = ep; 685c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* send_cq and recv_cq initialized below */ 686c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.srq = NULL; 687c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_wr = cdata->max_requests; 688c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 6890fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise case RPCRDMA_FRMR: { 6900fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise int depth = 7; 6910fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise 69215cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker /* Add room for frmr register and invalidate WRs. 69315cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker * 1. FRMR reg WR for head 69415cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker * 2. FRMR invalidate WR for head 6950fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise * 3. N FRMR reg WRs for pagelist 6960fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise * 4. N FRMR invalidate WRs for pagelist 69715cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker * 5. FRMR reg WR for tail 69815cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker * 6. FRMR invalidate WR for tail 69915cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker * 7. The RDMA_SEND WR 70015cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker */ 7010fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise 7020fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise /* Calculate N if the device max FRMR depth is smaller than 7030fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise * RPCRDMA_MAX_DATA_SEGS. 7040fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise */ 7050fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise if (ia->ri_max_frmr_depth < RPCRDMA_MAX_DATA_SEGS) { 7060fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise int delta = RPCRDMA_MAX_DATA_SEGS - 7070fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise ia->ri_max_frmr_depth; 7080fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise 7090fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise do { 7100fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise depth += 2; /* FRMR reg + invalidate */ 7110fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise delta -= ia->ri_max_frmr_depth; 7120fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise } while (delta > 0); 7130fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise 7140fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise } 7150fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise ep->rep_attr.cap.max_send_wr *= depth; 71615cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker if (ep->rep_attr.cap.max_send_wr > devattr.max_qp_wr) { 7170fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise cdata->max_requests = devattr.max_qp_wr / depth; 71815cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker if (!cdata->max_requests) 71915cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker return -EINVAL; 7200fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise ep->rep_attr.cap.max_send_wr = cdata->max_requests * 7210fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise depth; 72215cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker } 7233197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey break; 7240fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise } 725c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 726c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 727c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 728c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_recv_wr = cdata->max_requests; 729c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_sge = (cdata->padding ? 4 : 2); 730c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_recv_sge = 1; 731c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_inline_data = 0; 732c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.sq_sig_type = IB_SIGNAL_REQ_WR; 733c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.qp_type = IB_QPT_RC; 734c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.port_num = ~0; 735c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 736c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: requested max: dtos: send %d recv %d; " 737c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "iovs: send %d recv %d\n", 738c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, 739c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_wr, 740c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_recv_wr, 741c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_sge, 742c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_recv_sge); 743c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 744c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* set trigger for requesting send completion */ 745fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever ep->rep_cqinit = ep->rep_attr.cap.max_send_wr/2 - 1; 746c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_cqinit <= 2) 747c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_cqinit = 0; 748c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ INIT_CQCOUNT(ep); 749c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_ia = ia; 750c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ init_waitqueue_head(&ep->rep_connect_wait); 751254f91e2fa1f4cc18fd2eb9d5481888ffe126d5bChuck Lever INIT_DELAYED_WORK(&ep->rep_connect_worker, rpcrdma_connect_worker); 752c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 753fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever sendcq = ib_create_cq(ia->ri_id->device, rpcrdma_sendcq_upcall, 7541c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever rpcrdma_cq_async_error_upcall, ep, 755c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_wr + 1, 0); 756fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (IS_ERR(sendcq)) { 757fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rc = PTR_ERR(sendcq); 758fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever dprintk("RPC: %s: failed to create send CQ: %i\n", 759c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 760c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out1; 761c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 762c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 763fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rc = ib_req_notify_cq(sendcq, IB_CQ_NEXT_COMP); 764c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 765c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_req_notify_cq failed: %i\n", 766c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 767c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out2; 768c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 769c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 770fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever recvcq = ib_create_cq(ia->ri_id->device, rpcrdma_recvcq_upcall, 7711c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever rpcrdma_cq_async_error_upcall, ep, 772fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever ep->rep_attr.cap.max_recv_wr + 1, 0); 773fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (IS_ERR(recvcq)) { 774fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rc = PTR_ERR(recvcq); 775fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever dprintk("RPC: %s: failed to create recv CQ: %i\n", 776fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever __func__, rc); 777fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever goto out2; 778fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever } 779fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 780fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rc = ib_req_notify_cq(recvcq, IB_CQ_NEXT_COMP); 781fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (rc) { 782fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever dprintk("RPC: %s: ib_req_notify_cq failed: %i\n", 783fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever __func__, rc); 784fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever ib_destroy_cq(recvcq); 785fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever goto out2; 786fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever } 787fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 788fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever ep->rep_attr.send_cq = sendcq; 789fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever ep->rep_attr.recv_cq = recvcq; 790c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 791c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Initialize cma parameters */ 792c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 793c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* RPC/RDMA does not use private data */ 794c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.private_data = NULL; 795c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.private_data_len = 0; 796c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 797c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Client offers RDMA Read but does not initiate */ 798b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker ep->rep_remote_cma.initiator_depth = 0; 79903ff8821eb5ed168792667cfc3ddff903e97af99Chuck Lever if (devattr.max_qp_rd_atom > 32) /* arbitrary but <= 255 */ 800b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker ep->rep_remote_cma.responder_resources = 32; 801b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker else 802c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.responder_resources = devattr.max_qp_rd_atom; 803c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 804c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.retry_count = 7; 805c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.flow_control = 0; 806c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.rnr_retry_count = 0; 807c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 808c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 809c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 810c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out2: 811fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever err = ib_destroy_cq(sendcq); 8125d40a8a525c8165bafed233cf0f137e8d10d7e92Chuck Lever if (err) 8135d40a8a525c8165bafed233cf0f137e8d10d7e92Chuck Lever dprintk("RPC: %s: ib_destroy_cq returned %i\n", 8145d40a8a525c8165bafed233cf0f137e8d10d7e92Chuck Lever __func__, err); 815c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out1: 816c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 817c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 818c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 819c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 820c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * rpcrdma_ep_destroy 821c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 822c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Disconnect and destroy endpoint. After this, the only 823c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * valid operations on the ep are to free it (if dynamically 824c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * allocated) or re-create it. 825c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 8267f1d54191ed6fa0f79f584fe3ebf6519738e817fChuck Levervoid 827c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_destroy(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) 828c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 829c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 830c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 831c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: entering, connected is %d\n", 832c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, ep->rep_connected); 833c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 834254f91e2fa1f4cc18fd2eb9d5481888ffe126d5bChuck Lever cancel_delayed_work_sync(&ep->rep_connect_worker); 835254f91e2fa1f4cc18fd2eb9d5481888ffe126d5bChuck Lever 836c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ia->ri_id->qp) { 837c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_ep_disconnect(ep, ia); 838c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 839c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rpcrdma_ep_disconnect" 840c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " returned %i\n", __func__, rc); 841fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey rdma_destroy_qp(ia->ri_id); 842fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey ia->ri_id->qp = NULL; 843c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 844c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 845c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* padding - could be done in rpcrdma_buffer_destroy... */ 846c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_pad_mr) { 847c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_deregister_internal(ia, ep->rep_pad_mr, &ep->rep_pad); 848c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_pad_mr = NULL; 849c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 850c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 851fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rpcrdma_clean_cq(ep->rep_attr.recv_cq); 852fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rc = ib_destroy_cq(ep->rep_attr.recv_cq); 853fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (rc) 854fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever dprintk("RPC: %s: ib_destroy_cq returned %i\n", 855fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever __func__, rc); 856fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 857fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rpcrdma_clean_cq(ep->rep_attr.send_cq); 858fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rc = ib_destroy_cq(ep->rep_attr.send_cq); 859c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 860c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_destroy_cq returned %i\n", 861c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 862c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 863c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 864c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 865c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Connect unconnected endpoint. 866c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 867c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 868c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) 869c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 87073806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever struct rdma_cm_id *id, *old; 871c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc = 0; 872c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int retry_count = 0; 873c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 874c055551e97e1ca00781bc41523f829e05a8afed7Tom Talpey if (ep->rep_connected != 0) { 875c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_xprt *xprt; 876c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\retry: 877ec62f40d3505a643497d105c297093bb90afd44eChuck Lever dprintk("RPC: %s: reconnecting...\n", __func__); 878c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_ep_disconnect(ep, ia); 879c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc && rc != -ENOTCONN) 880c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rpcrdma_ep_disconnect" 881c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " status %i\n", __func__, rc); 882a7bc211ac926172ad20463afcf00ae7b9ebcd950Chuck Lever rpcrdma_flush_cqs(ep); 883c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 884c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ xprt = container_of(ia, struct rpcrdma_xprt, rx_ia); 885c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ id = rpcrdma_create_id(xprt, ia, 886c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (struct sockaddr *)&xprt->rx_data.addr); 887c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(id)) { 888ec62f40d3505a643497d105c297093bb90afd44eChuck Lever rc = -EHOSTUNREACH; 889c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 890c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 891c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* TEMP TEMP TEMP - fail if new device: 892c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Deregister/remarshal *all* requests! 893c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Close and recreate adapter, pd, etc! 894c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Re-determine all attributes still sane! 895c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * More stuff I haven't thought of! 896c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Rrrgh! 897c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 898c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ia->ri_id->device != id->device) { 899c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ printk("RPC: %s: can't reconnect on " 900c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "different device!\n", __func__); 901c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rdma_destroy_id(id); 902ec62f40d3505a643497d105c297093bb90afd44eChuck Lever rc = -ENETUNREACH; 903c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 904c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 905c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* END TEMP */ 906ec62f40d3505a643497d105c297093bb90afd44eChuck Lever rc = rdma_create_qp(id, ia->ri_pd, &ep->rep_attr); 907ec62f40d3505a643497d105c297093bb90afd44eChuck Lever if (rc) { 908ec62f40d3505a643497d105c297093bb90afd44eChuck Lever dprintk("RPC: %s: rdma_create_qp failed %i\n", 909ec62f40d3505a643497d105c297093bb90afd44eChuck Lever __func__, rc); 910ec62f40d3505a643497d105c297093bb90afd44eChuck Lever rdma_destroy_id(id); 911ec62f40d3505a643497d105c297093bb90afd44eChuck Lever rc = -ENETUNREACH; 912ec62f40d3505a643497d105c297093bb90afd44eChuck Lever goto out; 913ec62f40d3505a643497d105c297093bb90afd44eChuck Lever } 91473806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever 91573806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever write_lock(&ia->ri_qplock); 91673806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever old = ia->ri_id; 917c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_id = id; 91873806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever write_unlock(&ia->ri_qplock); 91973806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever 92073806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever rdma_destroy_qp(old); 92173806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever rdma_destroy_id(old); 922ec62f40d3505a643497d105c297093bb90afd44eChuck Lever } else { 923ec62f40d3505a643497d105c297093bb90afd44eChuck Lever dprintk("RPC: %s: connecting...\n", __func__); 924ec62f40d3505a643497d105c297093bb90afd44eChuck Lever rc = rdma_create_qp(ia->ri_id, ia->ri_pd, &ep->rep_attr); 925ec62f40d3505a643497d105c297093bb90afd44eChuck Lever if (rc) { 926ec62f40d3505a643497d105c297093bb90afd44eChuck Lever dprintk("RPC: %s: rdma_create_qp failed %i\n", 927ec62f40d3505a643497d105c297093bb90afd44eChuck Lever __func__, rc); 928ec62f40d3505a643497d105c297093bb90afd44eChuck Lever /* do not update ep->rep_connected */ 929ec62f40d3505a643497d105c297093bb90afd44eChuck Lever return -ENETUNREACH; 930ec62f40d3505a643497d105c297093bb90afd44eChuck Lever } 931c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 932c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 933c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = 0; 934c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 935c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rdma_connect(ia->ri_id, &ep->rep_remote_cma); 936c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 937c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_connect() failed with %i\n", 938c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 939c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 940c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 941c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 942c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wait_event_interruptible(ep->rep_connect_wait, ep->rep_connected != 0); 943c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 944c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 945c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Check state. A non-peer reject indicates no listener 946c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * (ECONNREFUSED), which may be a transient state. All 947c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * others indicate a transport condition which has already 948c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * undergone a best-effort. 949c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 950f64f9e719261a87818dd192a3a2352e5b20fbd0fJoe Perches if (ep->rep_connected == -ECONNREFUSED && 951f64f9e719261a87818dd192a3a2352e5b20fbd0fJoe Perches ++retry_count <= RDMA_CONNECT_RETRY_MAX) { 952c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: non-peer_reject, retry\n", __func__); 953c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto retry; 954c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 955c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_connected <= 0) { 956c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Sometimes, the only way to reliably connect to remote 957c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * CMs is to use same nonzero values for ORD and IRD. */ 958b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker if (retry_count++ <= RDMA_CONNECT_RETRY_MAX + 1 && 959b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker (ep->rep_remote_cma.responder_resources == 0 || 960b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker ep->rep_remote_cma.initiator_depth != 961b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker ep->rep_remote_cma.responder_resources)) { 962b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker if (ep->rep_remote_cma.responder_resources == 0) 963b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker ep->rep_remote_cma.responder_resources = 1; 964b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker ep->rep_remote_cma.initiator_depth = 965b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker ep->rep_remote_cma.responder_resources; 966c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto retry; 967b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker } 968c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ep->rep_connected; 969c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } else { 970c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: connected\n", __func__); 971c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 972c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 973c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out: 974c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 975c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = rc; 976c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 977c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 978c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 979c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 980c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * rpcrdma_ep_disconnect 981c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 982c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * This is separate from destroy to facilitate the ability 983c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * to reconnect without recreating the endpoint. 984c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 985c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * This call is not reentrant, and must not be made in parallel 986c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * on the same endpoint. 987c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 988c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 989c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_disconnect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) 990c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 991c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 992c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 993a7bc211ac926172ad20463afcf00ae7b9ebcd950Chuck Lever rpcrdma_flush_cqs(ep); 994c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rdma_disconnect(ia->ri_id); 995c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (!rc) { 996c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* returns without wait if not connected */ 997c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wait_event_interruptible(ep->rep_connect_wait, 998c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected != 1); 999c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: after wait, %sconnected\n", __func__, 1000c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (ep->rep_connected == 1) ? "still " : "dis"); 1001c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } else { 1002c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_disconnect %i\n", __func__, rc); 1003c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = rc; 1004c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1005c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 1006c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1007c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1008c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1009c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Initialize buffer memory 1010c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1011c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1012c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_buffer_create(struct rpcrdma_buffer *buf, struct rpcrdma_ep *ep, 1013c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia, struct rpcrdma_create_data_internal *cdata) 1014c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1015c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ char *p; 101665866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever size_t len, rlen, wlen; 1017c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int i, rc; 10188d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_mw *r; 1019c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1020c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_max_requests = cdata->max_requests; 1021c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_init(&buf->rb_lock); 1022c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ atomic_set(&buf->rb_credits, 1); 1023c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1024c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Need to allocate: 1025c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 1. arrays for send and recv pointers 1026c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 2. arrays of struct rpcrdma_req to fill in pointers 1027c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 3. array of struct rpcrdma_rep for replies 1028c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 4. padding, if any 10293197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey * 5. mw's, fmr's or frmr's, if any 1030c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Send/recv buffers in req/rep need to be registered 1031c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1032c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1033c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len = buf->rb_max_requests * 1034c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (sizeof(struct rpcrdma_req *) + sizeof(struct rpcrdma_rep *)); 1035c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len += cdata->padding; 1036c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 10373197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey case RPCRDMA_FRMR: 10383197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey len += buf->rb_max_requests * RPCRDMA_MAX_SEGS * 10393197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey sizeof(struct rpcrdma_mw); 10403197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey break; 1041c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 1042c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* TBD we are perhaps overallocating here */ 1043c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len += (buf->rb_max_requests + 1) * RPCRDMA_MAX_SEGS * 1044c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ sizeof(struct rpcrdma_mw); 1045c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1046c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 1047c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1048c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1049c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1050c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* allocate 1, 4 and 5 in one shot */ 1051c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ p = kzalloc(len, GFP_KERNEL); 1052c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (p == NULL) { 1053c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: req_t/rep_t/pad kzalloc(%zd) failed\n", 1054c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, len); 1055c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = -ENOMEM; 1056c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1057c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1058c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_pool = p; /* for freeing it later */ 1059c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1060c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_send_bufs = (struct rpcrdma_req **) p; 1061c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ p = (char *) &buf->rb_send_bufs[buf->rb_max_requests]; 1062c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_recv_bufs = (struct rpcrdma_rep **) p; 1063c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ p = (char *) &buf->rb_recv_bufs[buf->rb_max_requests]; 1064c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1065c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 1066c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Register the zeroed pad buffer, if any. 1067c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1068c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (cdata->padding) { 1069c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_register_internal(ia, p, cdata->padding, 1070c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &ep->rep_pad_mr, &ep->rep_pad); 1071c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1072c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1073c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1074c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ p += cdata->padding; 1075c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1076c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ INIT_LIST_HEAD(&buf->rb_mws); 10778d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey r = (struct rpcrdma_mw *)p; 1078c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 10793197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey case RPCRDMA_FRMR: 10803197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey for (i = buf->rb_max_requests * RPCRDMA_MAX_SEGS; i; i--) { 10813197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey r->r.frmr.fr_mr = ib_alloc_fast_reg_mr(ia->ri_pd, 10820fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise ia->ri_max_frmr_depth); 10833197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey if (IS_ERR(r->r.frmr.fr_mr)) { 10843197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey rc = PTR_ERR(r->r.frmr.fr_mr); 10853197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey dprintk("RPC: %s: ib_alloc_fast_reg_mr" 10863197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey " failed %i\n", __func__, rc); 10873197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey goto out; 10883197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey } 10890fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise r->r.frmr.fr_pgl = ib_alloc_fast_reg_page_list( 10900fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise ia->ri_id->device, 10910fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise ia->ri_max_frmr_depth); 10923197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey if (IS_ERR(r->r.frmr.fr_pgl)) { 10933197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey rc = PTR_ERR(r->r.frmr.fr_pgl); 10943197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey dprintk("RPC: %s: " 10953197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey "ib_alloc_fast_reg_page_list " 10963197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey "failed %i\n", __func__, rc); 10974034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews 10984034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews ib_dereg_mr(r->r.frmr.fr_mr); 10993197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey goto out; 11003197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey } 11013197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey list_add(&r->mw_list, &buf->rb_mws); 11023197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey ++r; 11033197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey } 11043197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey break; 1105c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 1106c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* TBD we are perhaps overallocating here */ 1107c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ for (i = (buf->rb_max_requests+1) * RPCRDMA_MAX_SEGS; i; i--) { 11088d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey static struct ib_fmr_attr fa = 11098d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey { RPCRDMA_MAX_DATA_SEGS, 1, PAGE_SHIFT }; 1110c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ r->r.fmr = ib_alloc_fmr(ia->ri_pd, 1111c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ, 1112c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &fa); 1113c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(r->r.fmr)) { 1114c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(r->r.fmr); 1115c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_alloc_fmr" 1116c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " failed %i\n", __func__, rc); 1117c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1118c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1119c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_add(&r->mw_list, &buf->rb_mws); 1120c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ++r; 1121c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1122c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1123c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 1124c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1125c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1126c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1127c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 1128c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Allocate/init the request/reply buffers. Doing this 1129c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * using kmalloc for now -- one for each buf. 1130c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 113165866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever wlen = 1 << fls(cdata->inline_wsize + sizeof(struct rpcrdma_req)); 113265866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever rlen = 1 << fls(cdata->inline_rsize + sizeof(struct rpcrdma_rep)); 113365866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever dprintk("RPC: %s: wlen = %zu, rlen = %zu\n", 113465866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever __func__, wlen, rlen); 113565866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever 1136c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ for (i = 0; i < buf->rb_max_requests; i++) { 1137c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_req *req; 1138c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep *rep; 1139c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 114065866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever req = kmalloc(wlen, GFP_KERNEL); 1141c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (req == NULL) { 1142c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: request buffer %d alloc" 1143c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " failed\n", __func__, i); 1144c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = -ENOMEM; 1145c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1146c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1147c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ memset(req, 0, sizeof(struct rpcrdma_req)); 1148c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_send_bufs[i] = req; 1149c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_send_bufs[i]->rl_buffer = buf; 1150c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1151c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_register_internal(ia, req->rl_base, 115265866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever wlen - offsetof(struct rpcrdma_req, rl_base), 1153c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_send_bufs[i]->rl_handle, 1154c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_send_bufs[i]->rl_iov); 1155c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1156c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1157c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 115865866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever buf->rb_send_bufs[i]->rl_size = wlen - 115965866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever sizeof(struct rpcrdma_req); 1160c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 116165866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever rep = kmalloc(rlen, GFP_KERNEL); 1162c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rep == NULL) { 1163c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: reply buffer %d alloc failed\n", 1164c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, i); 1165c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = -ENOMEM; 1166c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1167c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1168c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ memset(rep, 0, sizeof(struct rpcrdma_rep)); 1169c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_recv_bufs[i] = rep; 1170c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_recv_bufs[i]->rr_buffer = buf; 1171c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1172c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_register_internal(ia, rep->rr_base, 117365866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever rlen - offsetof(struct rpcrdma_rep, rr_base), 1174c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_recv_bufs[i]->rr_handle, 1175c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_recv_bufs[i]->rr_iov); 1176c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1177c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1178c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1179c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1180c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: max_requests %d\n", 1181c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, buf->rb_max_requests); 1182c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* done */ 1183c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 1184c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out: 1185c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_buffer_destroy(buf); 1186c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 1187c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1188c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1189c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1190c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Unregister and destroy buffer memory. Need to deal with 1191c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * partial initialization, so it's callable from failed create. 1192c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Must be called before destroying endpoint, as registrations 1193c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * reference it. 1194c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1195c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void 1196c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf) 1197c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1198c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc, i; 1199c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = rdmab_to_ia(buf); 12008d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_mw *r; 1201c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1202c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* clean up in reverse order from create 1203c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 1. recv mr memory (mr free, then kfree) 1204c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 2. send mr memory (mr free, then kfree) 1205c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 3. padding (if any) [moved to rpcrdma_ep_destroy] 1206c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 4. arrays 1207c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1208c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: entering\n", __func__); 1209c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1210c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ for (i = 0; i < buf->rb_max_requests; i++) { 1211c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (buf->rb_recv_bufs && buf->rb_recv_bufs[i]) { 1212c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_deregister_internal(ia, 1213c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_recv_bufs[i]->rr_handle, 1214c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_recv_bufs[i]->rr_iov); 1215c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ kfree(buf->rb_recv_bufs[i]); 1216c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1217c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (buf->rb_send_bufs && buf->rb_send_bufs[i]) { 1218c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_deregister_internal(ia, 1219c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_send_bufs[i]->rl_handle, 1220c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_send_bufs[i]->rl_iov); 1221c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ kfree(buf->rb_send_bufs[i]); 1222c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1223c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1224c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 12254034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews while (!list_empty(&buf->rb_mws)) { 12264034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews r = list_entry(buf->rb_mws.next, 12274034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews struct rpcrdma_mw, mw_list); 12284034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews list_del(&r->mw_list); 12294034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews switch (ia->ri_memreg_strategy) { 12304034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews case RPCRDMA_FRMR: 12314034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews rc = ib_dereg_mr(r->r.frmr.fr_mr); 12324034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews if (rc) 12334034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews dprintk("RPC: %s:" 12344034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews " ib_dereg_mr" 12354034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews " failed %i\n", 12364034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews __func__, rc); 12374034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews ib_free_fast_reg_page_list(r->r.frmr.fr_pgl); 12384034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews break; 12394034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews case RPCRDMA_MTHCAFMR: 12404034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews rc = ib_dealloc_fmr(r->r.fmr); 12414034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews if (rc) 12424034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews dprintk("RPC: %s:" 12434034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews " ib_dealloc_fmr" 12444034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews " failed %i\n", 12454034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews __func__, rc); 12464034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews break; 12474034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews default: 12484034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews break; 12494034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews } 12504034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews } 12514034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews 1252c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ kfree(buf->rb_pool); 1253c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1254c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1255c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1256c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Get a set of request/reply buffers. 1257c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 1258c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Reply buffer (if needed) is attached to send buffer upon return. 1259c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Rule: 1260c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * rb_send_index and rb_recv_index MUST always be pointing to the 1261c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * *next* available buffer (non-NULL). They are incremented after 1262c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * removing buffers, and decremented *before* returning them. 1263c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1264c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\struct rpcrdma_req * 1265c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_buffer_get(struct rpcrdma_buffer *buffers) 1266c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1267c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_req *req; 1268c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 12698d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int i; 12708d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_mw *r; 1271c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1272c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&buffers->rb_lock, flags); 1273c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (buffers->rb_send_index == buffers->rb_max_requests) { 1274c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&buffers->rb_lock, flags); 1275c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: out of request buffers\n", __func__); 1276c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return ((struct rpcrdma_req *)NULL); 1277c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1278c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1279c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req = buffers->rb_send_bufs[buffers->rb_send_index]; 1280c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (buffers->rb_send_index < buffers->rb_recv_index) { 1281c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: %d extra receives outstanding (ok)\n", 1282c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, 1283c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_recv_index - buffers->rb_send_index); 1284c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply = NULL; 1285c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } else { 1286c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply = buffers->rb_recv_bufs[buffers->rb_recv_index]; 1287c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_recv_bufs[buffers->rb_recv_index++] = NULL; 1288c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1289c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_send_bufs[buffers->rb_send_index++] = NULL; 1290c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (!list_empty(&buffers->rb_mws)) { 12918d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey i = RPCRDMA_MAX_SEGS - 1; 1292c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ do { 1293c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ r = list_entry(buffers->rb_mws.next, 1294c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_mw, mw_list); 1295c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_del(&r->mw_list); 1296c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_segments[i].mr_chunk.rl_mw = r; 1297c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } while (--i >= 0); 1298c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1299c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&buffers->rb_lock, flags); 1300c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return req; 1301c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1302c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1303c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1304c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Put request/reply buffers back into pool. 1305c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Pre-decrement counter/array index. 1306c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1307c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void 1308c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_buffer_put(struct rpcrdma_req *req) 1309c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1310c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_buffer *buffers = req->rl_buffer; 1311c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = rdmab_to_ia(buffers); 1312c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int i; 1313c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 1314c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1315c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&buffers->rb_lock, flags); 1316c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_send_bufs[--buffers->rb_send_index] = req; 1317c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_niovs = 0; 1318c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (req->rl_reply) { 1319c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_recv_bufs[--buffers->rb_recv_index] = req->rl_reply; 1320c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply->rr_func = NULL; 1321c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply = NULL; 1322c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1323c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 13243197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey case RPCRDMA_FRMR: 1325c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 1326c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 1327c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Cycle mw's back in reverse order, and "spin" them. 1328c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * This delays and scrambles reuse as much as possible. 1329c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1330c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ i = 1; 1331c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ do { 1332c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_mw **mw; 1333c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ mw = &req->rl_segments[i].mr_chunk.rl_mw; 1334c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_add_tail(&(*mw)->mw_list, &buffers->rb_mws); 1335c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ *mw = NULL; 1336c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } while (++i < RPCRDMA_MAX_SEGS); 1337c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_add_tail(&req->rl_segments[0].mr_chunk.rl_mw->mw_list, 1338c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buffers->rb_mws); 1339c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_segments[0].mr_chunk.rl_mw = NULL; 1340c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1341c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 1342c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1343c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1344c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&buffers->rb_lock, flags); 1345c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1346c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1347c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1348c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Recover reply buffers from pool. 1349c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * This happens when recovering from error conditions. 1350c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Post-increment counter/array index. 1351c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1352c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void 1353c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_recv_buffer_get(struct rpcrdma_req *req) 1354c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1355c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_buffer *buffers = req->rl_buffer; 1356c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 1357c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1358c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (req->rl_iov.length == 0) /* special case xprt_rdma_allocate() */ 1359c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers = ((struct rpcrdma_req *) buffers)->rl_buffer; 1360c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&buffers->rb_lock, flags); 1361c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (buffers->rb_recv_index < buffers->rb_max_requests) { 1362c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply = buffers->rb_recv_bufs[buffers->rb_recv_index]; 1363c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_recv_bufs[buffers->rb_recv_index++] = NULL; 1364c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1365c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&buffers->rb_lock, flags); 1366c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1367c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1368c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1369c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Put reply buffers back into pool when not attached to 1370b45ccfd25d506e83d9ecf93d0ac7edf031d35d2fChuck Lever * request. This happens in error conditions. 1371c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1372c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void 1373c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep) 1374c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1375c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_buffer *buffers = rep->rr_buffer; 1376c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 1377c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1378c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_func = NULL; 1379c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&buffers->rb_lock, flags); 1380c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_recv_bufs[--buffers->rb_recv_index] = rep; 1381c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&buffers->rb_lock, flags); 1382c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1383c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1384c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1385c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Wrappers for internal-use kmalloc memory registration, used by buffer code. 1386c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1387c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1388c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1389c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_register_internal(struct rpcrdma_ia *ia, void *va, int len, 1390c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_mr **mrp, struct ib_sge *iov) 1391c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1392c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_phys_buf ipb; 1393c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_mr *mr; 1394c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 1395c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1396c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 1397c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * All memory passed here was kmalloc'ed, therefore phys-contiguous. 1398c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1399c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ iov->addr = ib_dma_map_single(ia->ri_id->device, 1400c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ va, len, DMA_BIDIRECTIONAL); 1401bf858ab0adcb36fc60a1d941749a7dd0a9bf3d8aYan Burman if (ib_dma_mapping_error(ia->ri_id->device, iov->addr)) 1402bf858ab0adcb36fc60a1d941749a7dd0a9bf3d8aYan Burman return -ENOMEM; 1403bf858ab0adcb36fc60a1d941749a7dd0a9bf3d8aYan Burman 1404c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ iov->length = len; 1405c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1406bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey if (ia->ri_have_dma_lkey) { 1407bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey *mrp = NULL; 1408bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey iov->lkey = ia->ri_dma_lkey; 1409bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey return 0; 1410bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey } else if (ia->ri_bind_mem != NULL) { 1411c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ *mrp = NULL; 1412c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ iov->lkey = ia->ri_bind_mem->lkey; 1413c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 1414c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1415c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1416c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ipb.addr = iov->addr; 1417c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ipb.size = iov->length; 1418c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ mr = ib_reg_phys_mr(ia->ri_pd, &ipb, 1, 1419c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ IB_ACCESS_LOCAL_WRITE, &iov->addr); 1420c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1421c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: phys convert: 0x%llx " 1422c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "registered 0x%llx length %d\n", 1423a56daeb7d5c9a05b1cb52ae4bcca05fb6545656eAndrew Morton __func__, (unsigned long long)ipb.addr, 1424a56daeb7d5c9a05b1cb52ae4bcca05fb6545656eAndrew Morton (unsigned long long)iov->addr, len); 1425c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1426c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(mr)) { 1427c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ *mrp = NULL; 1428c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(mr); 1429c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: failed with %i\n", __func__, rc); 1430c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } else { 1431c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ *mrp = mr; 1432c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ iov->lkey = mr->lkey; 1433c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = 0; 1434c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1435c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1436c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 1437c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1438c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1439c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1440c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_deregister_internal(struct rpcrdma_ia *ia, 1441c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_mr *mr, struct ib_sge *iov) 1442c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1443c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 1444c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1445c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_unmap_single(ia->ri_id->device, 1446c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ iov->addr, iov->length, DMA_BIDIRECTIONAL); 1447c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1448c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (NULL == mr) 1449c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 1450c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1451c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_dereg_mr(mr); 1452c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1453c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_dereg_mr failed %i\n", __func__, rc); 1454c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 1455c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1456c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1457c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1458c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Wrappers for chunk registration, shared by read/write chunk code. 1459c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1460c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1461c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 1462c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_map_one(struct rpcrdma_ia *ia, struct rpcrdma_mr_seg *seg, int writing) 1463c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1464c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dir = writing ? DMA_FROM_DEVICE : DMA_TO_DEVICE; 1465c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dmalen = seg->mr_len; 1466c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (seg->mr_page) 1467c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dma = ib_dma_map_page(ia->ri_id->device, 1468c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_page, offset_in_page(seg->mr_offset), 1469c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dmalen, seg->mr_dir); 1470c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ else 1471c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dma = ib_dma_map_single(ia->ri_id->device, 1472c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_offset, 1473c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dmalen, seg->mr_dir); 14745c635e09cec0feeeb310968e51dad01040244851Tom Tucker if (ib_dma_mapping_error(ia->ri_id->device, seg->mr_dma)) { 14755c635e09cec0feeeb310968e51dad01040244851Tom Tucker dprintk("RPC: %s: mr_dma %llx mr_offset %p mr_dma_len %zu\n", 14765c635e09cec0feeeb310968e51dad01040244851Tom Tucker __func__, 1477986d4abbddf9e76184f6cabf66654ea8e61bcde5Randy Dunlap (unsigned long long)seg->mr_dma, 1478986d4abbddf9e76184f6cabf66654ea8e61bcde5Randy Dunlap seg->mr_offset, seg->mr_dmalen); 14795c635e09cec0feeeb310968e51dad01040244851Tom Tucker } 1480c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1481c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1482c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 1483c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_unmap_one(struct rpcrdma_ia *ia, struct rpcrdma_mr_seg *seg) 1484c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1485c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (seg->mr_page) 1486c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_unmap_page(ia->ri_id->device, 1487c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dma, seg->mr_dmalen, seg->mr_dir); 1488c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ else 1489c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_unmap_single(ia->ri_id->device, 1490c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dma, seg->mr_dmalen, seg->mr_dir); 1491c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1492c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 14938d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeystatic int 14943197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpeyrpcrdma_register_frmr_external(struct rpcrdma_mr_seg *seg, 14953197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey int *nsegs, int writing, struct rpcrdma_ia *ia, 14963197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey struct rpcrdma_xprt *r_xprt) 14973197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey{ 14983197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey struct rpcrdma_mr_seg *seg1 = seg; 14990dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever struct rpcrdma_mw *mw = seg1->mr_chunk.rl_mw; 15000dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever struct rpcrdma_frmr *frmr = &mw->r.frmr; 15010dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever struct ib_mr *mr = frmr->fr_mr; 15025c635e09cec0feeeb310968e51dad01040244851Tom Tucker struct ib_send_wr invalidate_wr, frmr_wr, *bad_wr, *post_wr; 15035c635e09cec0feeeb310968e51dad01040244851Tom Tucker 15043197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey u8 key; 15053197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey int len, pageoff; 15063197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey int i, rc; 15079b78145c0f280d4f01c460d6251eab2584181fa9Tom Tucker int seg_len; 15089b78145c0f280d4f01c460d6251eab2584181fa9Tom Tucker u64 pa; 15099b78145c0f280d4f01c460d6251eab2584181fa9Tom Tucker int page_no; 15103197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 15113197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey pageoff = offset_in_page(seg1->mr_offset); 15123197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey seg1->mr_offset -= pageoff; /* start of page */ 15133197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey seg1->mr_len += pageoff; 15143197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey len = -pageoff; 15150fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise if (*nsegs > ia->ri_max_frmr_depth) 15160fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise *nsegs = ia->ri_max_frmr_depth; 15179b78145c0f280d4f01c460d6251eab2584181fa9Tom Tucker for (page_no = i = 0; i < *nsegs;) { 15183197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey rpcrdma_map_one(ia, seg, writing); 15199b78145c0f280d4f01c460d6251eab2584181fa9Tom Tucker pa = seg->mr_dma; 15209b78145c0f280d4f01c460d6251eab2584181fa9Tom Tucker for (seg_len = seg->mr_len; seg_len > 0; seg_len -= PAGE_SIZE) { 15210dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever frmr->fr_pgl->page_list[page_no++] = pa; 15229b78145c0f280d4f01c460d6251eab2584181fa9Tom Tucker pa += PAGE_SIZE; 15239b78145c0f280d4f01c460d6251eab2584181fa9Tom Tucker } 15243197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey len += seg->mr_len; 15253197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey ++seg; 15263197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey ++i; 15273197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey /* Check for holes */ 15283197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey if ((i < *nsegs && offset_in_page(seg->mr_offset)) || 15293197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len)) 15303197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey break; 15313197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey } 15323197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey dprintk("RPC: %s: Using frmr %p to map %d segments\n", 15330dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever __func__, mw, i); 15343197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 15350dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever if (unlikely(frmr->fr_state == FRMR_IS_VALID)) { 15365c635e09cec0feeeb310968e51dad01040244851Tom Tucker dprintk("RPC: %s: frmr %x left valid, posting invalidate.\n", 15370dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever __func__, mr->rkey); 15385c635e09cec0feeeb310968e51dad01040244851Tom Tucker /* Invalidate before using. */ 15395c635e09cec0feeeb310968e51dad01040244851Tom Tucker memset(&invalidate_wr, 0, sizeof invalidate_wr); 15400dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever invalidate_wr.wr_id = (unsigned long)(void *)mw; 15415c635e09cec0feeeb310968e51dad01040244851Tom Tucker invalidate_wr.next = &frmr_wr; 15425c635e09cec0feeeb310968e51dad01040244851Tom Tucker invalidate_wr.opcode = IB_WR_LOCAL_INV; 15435c635e09cec0feeeb310968e51dad01040244851Tom Tucker invalidate_wr.send_flags = IB_SEND_SIGNALED; 15440dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever invalidate_wr.ex.invalidate_rkey = mr->rkey; 15455c635e09cec0feeeb310968e51dad01040244851Tom Tucker DECR_CQCOUNT(&r_xprt->rx_ep); 15465c635e09cec0feeeb310968e51dad01040244851Tom Tucker post_wr = &invalidate_wr; 15475c635e09cec0feeeb310968e51dad01040244851Tom Tucker } else 15485c635e09cec0feeeb310968e51dad01040244851Tom Tucker post_wr = &frmr_wr; 15495c635e09cec0feeeb310968e51dad01040244851Tom Tucker 15503197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey /* Prepare FRMR WR */ 15513197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey memset(&frmr_wr, 0, sizeof frmr_wr); 15520dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever frmr_wr.wr_id = (unsigned long)(void *)mw; 15533197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey frmr_wr.opcode = IB_WR_FAST_REG_MR; 15545c635e09cec0feeeb310968e51dad01040244851Tom Tucker frmr_wr.send_flags = IB_SEND_SIGNALED; 15557a8b80eb38b248cfdf84048dad12073d5cfba3e6Steve Wise frmr_wr.wr.fast_reg.iova_start = seg1->mr_dma; 15560dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever frmr_wr.wr.fast_reg.page_list = frmr->fr_pgl; 15579b78145c0f280d4f01c460d6251eab2584181fa9Tom Tucker frmr_wr.wr.fast_reg.page_list_len = page_no; 15583197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey frmr_wr.wr.fast_reg.page_shift = PAGE_SHIFT; 15599b78145c0f280d4f01c460d6251eab2584181fa9Tom Tucker frmr_wr.wr.fast_reg.length = page_no << PAGE_SHIFT; 1560c977dea22708688eae31774f70126c97aa4dfe83Chuck Lever if (frmr_wr.wr.fast_reg.length < len) { 15615fc83f470d8ada25927701512cf94a53dab6c4c8Chuck Lever rc = -EIO; 15625fc83f470d8ada25927701512cf94a53dab6c4c8Chuck Lever goto out_err; 1563c977dea22708688eae31774f70126c97aa4dfe83Chuck Lever } 1564c977dea22708688eae31774f70126c97aa4dfe83Chuck Lever 1565c977dea22708688eae31774f70126c97aa4dfe83Chuck Lever /* Bump the key */ 15660dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever key = (u8)(mr->rkey & 0x000000FF); 15670dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever ib_update_fast_reg_key(mr, ++key); 1568c977dea22708688eae31774f70126c97aa4dfe83Chuck Lever 15693197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey frmr_wr.wr.fast_reg.access_flags = (writing ? 157068743082b560067e3e93eab8b2568f238e486865Vu Pham IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE : 157168743082b560067e3e93eab8b2568f238e486865Vu Pham IB_ACCESS_REMOTE_READ); 15720dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever frmr_wr.wr.fast_reg.rkey = mr->rkey; 15733197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey DECR_CQCOUNT(&r_xprt->rx_ep); 15743197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 15755c635e09cec0feeeb310968e51dad01040244851Tom Tucker rc = ib_post_send(ia->ri_id->qp, post_wr, &bad_wr); 15763197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 15773197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey if (rc) { 15783197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey dprintk("RPC: %s: failed ib_post_send for register," 15793197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey " status %i\n", __func__, rc); 1580c93e986a295d537589efd0504f36ca952bd1a5beChuck Lever ib_update_fast_reg_key(mr, --key); 15815fc83f470d8ada25927701512cf94a53dab6c4c8Chuck Lever goto out_err; 15823197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey } else { 15830dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever seg1->mr_rkey = mr->rkey; 15843197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey seg1->mr_base = seg1->mr_dma + pageoff; 15853197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey seg1->mr_nsegs = i; 15863197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey seg1->mr_len = len; 15873197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey } 15883197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey *nsegs = i; 15895fc83f470d8ada25927701512cf94a53dab6c4c8Chuck Lever return 0; 15905fc83f470d8ada25927701512cf94a53dab6c4c8Chuck Leverout_err: 15915fc83f470d8ada25927701512cf94a53dab6c4c8Chuck Lever while (i--) 15925fc83f470d8ada25927701512cf94a53dab6c4c8Chuck Lever rpcrdma_unmap_one(ia, --seg); 15933197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey return rc; 15943197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey} 15953197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 15963197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpeystatic int 15973197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpeyrpcrdma_deregister_frmr_external(struct rpcrdma_mr_seg *seg, 15983197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey struct rpcrdma_ia *ia, struct rpcrdma_xprt *r_xprt) 15993197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey{ 16003197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey struct rpcrdma_mr_seg *seg1 = seg; 16013197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey struct ib_send_wr invalidate_wr, *bad_wr; 16023197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey int rc; 16033197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 16043197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey memset(&invalidate_wr, 0, sizeof invalidate_wr); 16055c635e09cec0feeeb310968e51dad01040244851Tom Tucker invalidate_wr.wr_id = (unsigned long)(void *)seg1->mr_chunk.rl_mw; 16063197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey invalidate_wr.opcode = IB_WR_LOCAL_INV; 16075c635e09cec0feeeb310968e51dad01040244851Tom Tucker invalidate_wr.send_flags = IB_SEND_SIGNALED; 16083197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey invalidate_wr.ex.invalidate_rkey = seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey; 16093197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey DECR_CQCOUNT(&r_xprt->rx_ep); 16103197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 161173806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever read_lock(&ia->ri_qplock); 161273806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever while (seg1->mr_nsegs--) 161373806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever rpcrdma_unmap_one(ia, seg++); 16143197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey rc = ib_post_send(ia->ri_id->qp, &invalidate_wr, &bad_wr); 161573806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever read_unlock(&ia->ri_qplock); 16163197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey if (rc) 16173197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey dprintk("RPC: %s: failed ib_post_send for invalidate," 16183197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey " status %i\n", __func__, rc); 16193197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey return rc; 16203197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey} 16213197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 16223197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpeystatic int 16238d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeyrpcrdma_register_fmr_external(struct rpcrdma_mr_seg *seg, 16248d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int *nsegs, int writing, struct rpcrdma_ia *ia) 16258d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey{ 16268d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_mr_seg *seg1 = seg; 16278d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey u64 physaddrs[RPCRDMA_MAX_DATA_SEGS]; 16288d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int len, pageoff, i, rc; 16298d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 16308d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey pageoff = offset_in_page(seg1->mr_offset); 16318d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_offset -= pageoff; /* start of page */ 16328d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_len += pageoff; 16338d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey len = -pageoff; 16348d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (*nsegs > RPCRDMA_MAX_DATA_SEGS) 16358d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey *nsegs = RPCRDMA_MAX_DATA_SEGS; 16368d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey for (i = 0; i < *nsegs;) { 16378d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_map_one(ia, seg, writing); 16388d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey physaddrs[i] = seg->mr_dma; 16398d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey len += seg->mr_len; 16408d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey ++seg; 16418d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey ++i; 16428d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey /* Check for holes */ 16438d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if ((i < *nsegs && offset_in_page(seg->mr_offset)) || 16448d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len)) 16458d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey break; 16468d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } 16478d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = ib_map_phys_fmr(seg1->mr_chunk.rl_mw->r.fmr, 16488d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey physaddrs, i, seg1->mr_dma); 16498d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (rc) { 16508d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey dprintk("RPC: %s: failed ib_map_phys_fmr " 16518d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey "%u@0x%llx+%i (%d)... status %i\n", __func__, 16528d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey len, (unsigned long long)seg1->mr_dma, 16538d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey pageoff, i, rc); 16548d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey while (i--) 16558d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_unmap_one(ia, --seg); 16568d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } else { 16578d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_rkey = seg1->mr_chunk.rl_mw->r.fmr->rkey; 16588d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_base = seg1->mr_dma + pageoff; 16598d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_nsegs = i; 16608d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_len = len; 16618d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } 16628d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey *nsegs = i; 16638d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey return rc; 16648d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey} 16658d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 16668d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeystatic int 16678d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeyrpcrdma_deregister_fmr_external(struct rpcrdma_mr_seg *seg, 16688d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_ia *ia) 16698d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey{ 16708d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_mr_seg *seg1 = seg; 16718d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey LIST_HEAD(l); 16728d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int rc; 16738d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 16748d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey list_add(&seg1->mr_chunk.rl_mw->r.fmr->list, &l); 16758d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = ib_unmap_fmr(&l); 167673806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever read_lock(&ia->ri_qplock); 16778d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey while (seg1->mr_nsegs--) 16788d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_unmap_one(ia, seg++); 167973806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever read_unlock(&ia->ri_qplock); 16808d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (rc) 16818d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey dprintk("RPC: %s: failed ib_unmap_fmr," 16828d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey " status %i\n", __func__, rc); 16838d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey return rc; 16848d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey} 16858d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 1686c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1687c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_register_external(struct rpcrdma_mr_seg *seg, 1688c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int nsegs, int writing, struct rpcrdma_xprt *r_xprt) 1689c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1690c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = &r_xprt->rx_ia; 1691c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc = 0; 1692c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1693c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 1694c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1695c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#if RPCRDMA_PERSISTENT_REGISTRATION 1696c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_ALLPHYSICAL: 1697c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_map_one(ia, seg, writing); 1698c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_rkey = ia->ri_bind_mem->rkey; 1699c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_base = seg->mr_dma; 1700c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_nsegs = 1; 1701c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ nsegs = 1; 1702c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1703c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#endif 1704c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 17053197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey /* Registration using frmr registration */ 17063197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey case RPCRDMA_FRMR: 17073197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey rc = rpcrdma_register_frmr_external(seg, &nsegs, writing, ia, r_xprt); 17083197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey break; 17093197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 17108d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey /* Registration using fmr memory registration */ 1711c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 17128d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = rpcrdma_register_fmr_external(seg, &nsegs, writing, ia); 1713c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1714c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1715c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 17160ac531c1832318efa3dc3d723e356a7e09330e80Chuck Lever return -1; 1717c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1718c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1719c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return -1; 1720c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1721c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return nsegs; 1722c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1723c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1724c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1725c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_deregister_external(struct rpcrdma_mr_seg *seg, 172613c9ff8f673862b69e795ea99a237b461c557eb3Chuck Lever struct rpcrdma_xprt *r_xprt) 1727c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1728c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = &r_xprt->rx_ia; 1729c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int nsegs = seg->mr_nsegs, rc; 1730c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1731c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 1732c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1733c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#if RPCRDMA_PERSISTENT_REGISTRATION 1734c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_ALLPHYSICAL: 173573806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever read_lock(&ia->ri_qplock); 1736c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_unmap_one(ia, seg); 173773806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever read_unlock(&ia->ri_qplock); 1738c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1739c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#endif 1740c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 17413197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey case RPCRDMA_FRMR: 17423197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey rc = rpcrdma_deregister_frmr_external(seg, ia, r_xprt); 17433197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey break; 17443197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 1745c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 17468d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = rpcrdma_deregister_fmr_external(seg, ia); 1747c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1748c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1749c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 1750c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1751c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1752c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return nsegs; 1753c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1754c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1755c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1756c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Prepost any receive buffer, then post send. 1757c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 1758c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Receive buffer is donated to hardware, reclaimed upon recv completion. 1759c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1760c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1761c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_post(struct rpcrdma_ia *ia, 1762c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ep *ep, 1763c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_req *req) 1764c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1765c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_send_wr send_wr, *send_wr_fail; 1766c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep *rep = req->rl_reply; 1767c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 1768c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1769c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rep) { 1770c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_ep_post_recv(ia, ep, rep); 1771c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1772c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1773c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply = NULL; 1774c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1775c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1776c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.next = NULL; 1777c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.wr_id = 0ULL; /* no send cookie */ 1778c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.sg_list = req->rl_send_iov; 1779c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.num_sge = req->rl_niovs; 1780c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.opcode = IB_WR_SEND; 1781c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (send_wr.num_sge == 4) /* no need to sync any pad (constant) */ 1782c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_sync_single_for_device(ia->ri_id->device, 1783c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_send_iov[3].addr, req->rl_send_iov[3].length, 1784c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ DMA_TO_DEVICE); 1785c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_sync_single_for_device(ia->ri_id->device, 1786c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_send_iov[1].addr, req->rl_send_iov[1].length, 1787c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ DMA_TO_DEVICE); 1788c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_sync_single_for_device(ia->ri_id->device, 1789c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_send_iov[0].addr, req->rl_send_iov[0].length, 1790c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ DMA_TO_DEVICE); 1791c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1792c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (DECR_CQCOUNT(ep) > 0) 1793c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.send_flags = 0; 1794c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ else { /* Provider must take a send completion every now and then */ 1795c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ INIT_CQCOUNT(ep); 1796c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.send_flags = IB_SEND_SIGNALED; 1797c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1798c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1799c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_post_send(ia->ri_id->qp, &send_wr, &send_wr_fail); 1800c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1801c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_post_send returned %i\n", __func__, 1802c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc); 1803c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out: 1804c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 1805c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1806c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1807c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1808c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * (Re)post a receive buffer. 1809c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1810c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1811c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_post_recv(struct rpcrdma_ia *ia, 1812c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ep *ep, 1813c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep *rep) 1814c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1815c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_recv_wr recv_wr, *recv_wr_fail; 1816c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 1817c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1818c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ recv_wr.next = NULL; 1819c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ recv_wr.wr_id = (u64) (unsigned long) rep; 1820c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ recv_wr.sg_list = &rep->rr_iov; 1821c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ recv_wr.num_sge = 1; 1822c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1823c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_sync_single_for_cpu(ia->ri_id->device, 1824c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_iov.addr, rep->rr_iov.length, DMA_BIDIRECTIONAL); 1825c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1826c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_post_recv(ia->ri_id->qp, &recv_wr, &recv_wr_fail); 1827c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1828c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1829c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_post_recv returned %i\n", __func__, 1830c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc); 1831c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 1832c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 183343e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever 183443e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever/* Physical mapping means one Read/Write list entry per-page. 183543e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever * All list entries must fit within an inline buffer 183643e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever * 183743e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever * NB: The server must return a Write list for NFS READ, 183843e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever * which has the same constraint. Factor in the inline 183943e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever * rsize as well. 184043e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever */ 184143e95988178ed70a878a5be6be9ad248342dbf7dChuck Leverstatic size_t 184243e95988178ed70a878a5be6be9ad248342dbf7dChuck Leverrpcrdma_physical_max_payload(struct rpcrdma_xprt *r_xprt) 184343e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever{ 184443e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever struct rpcrdma_create_data_internal *cdata = &r_xprt->rx_data; 184543e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever unsigned int inline_size, pages; 184643e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever 184743e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever inline_size = min_t(unsigned int, 184843e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever cdata->inline_wsize, cdata->inline_rsize); 184943e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever inline_size -= RPCRDMA_HDRLEN_MIN; 185043e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever pages = inline_size / sizeof(struct rpcrdma_segment); 185143e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever return pages << PAGE_SHIFT; 185243e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever} 185343e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever 185443e95988178ed70a878a5be6be9ad248342dbf7dChuck Leverstatic size_t 185543e95988178ed70a878a5be6be9ad248342dbf7dChuck Leverrpcrdma_mr_max_payload(struct rpcrdma_xprt *r_xprt) 185643e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever{ 185743e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever return RPCRDMA_MAX_DATA_SEGS << PAGE_SHIFT; 185843e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever} 185943e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever 186043e95988178ed70a878a5be6be9ad248342dbf7dChuck Leversize_t 186143e95988178ed70a878a5be6be9ad248342dbf7dChuck Leverrpcrdma_max_payload(struct rpcrdma_xprt *r_xprt) 186243e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever{ 186343e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever size_t result; 186443e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever 186543e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever switch (r_xprt->rx_ia.ri_memreg_strategy) { 186643e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever case RPCRDMA_ALLPHYSICAL: 186743e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever result = rpcrdma_physical_max_payload(r_xprt); 186843e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever break; 186943e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever default: 187043e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever result = rpcrdma_mr_max_payload(r_xprt); 187143e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever } 187243e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever return result; 187343e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever} 1874