1f58851e6b0f148fb4b2a1c6f70beb2f125863c0f\"Talpey, Thomas\/* 2c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved. 3c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 4c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * This software is available to you under a choice of one of two 5c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * licenses. You may choose to be licensed under the terms of the GNU 6c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * General Public License (GPL) Version 2, available from the file 7c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * COPYING in the main directory of this source tree, or the BSD-type 8c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * license below: 9c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 10c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Redistribution and use in source and binary forms, with or without 11c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * modification, are permitted provided that the following conditions 12c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * are met: 13c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 14c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Redistributions of source code must retain the above copyright 15c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * notice, this list of conditions and the following disclaimer. 16c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 17c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Redistributions in binary form must reproduce the above 18c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * copyright notice, this list of conditions and the following 19c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * disclaimer in the documentation and/or other materials provided 20c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * with the distribution. 21c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 22c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Neither the name of the Network Appliance, Inc. nor the names of 23c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * its contributors may be used to endorse or promote products 24c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * derived from this software without specific prior written 25c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * permission. 26c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 27c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 28c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 29c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 30c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 31c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 32c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 33c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 34c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 35c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 36c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 37c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 38f58851e6b0f148fb4b2a1c6f70beb2f125863c0f\"Talpey, Thomas\ */ 39f58851e6b0f148fb4b2a1c6f70beb2f125863c0f\"Talpey, Thomas\ 40c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 41c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * verbs.c 42c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 43c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Encapsulates the major functions managing: 44c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o adapters 45c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o endpoints 46c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o connections 47c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o buffer memory 48c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 49c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 50a6b7a407865aab9f849dd99a71072b7cd1175116Alexey Dobriyan#include <linux/interrupt.h> 515a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 5265866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever#include <asm/bitops.h> 53c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 54f58851e6b0f148fb4b2a1c6f70beb2f125863c0f\"Talpey, Thomas\#include "xprt_rdma.h" 55f58851e6b0f148fb4b2a1c6f70beb2f125863c0f\"Talpey, Thomas\ 56c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 57c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Globals/Macros 58c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 59c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 60c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#ifdef RPC_DEBUG 61c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\# define RPCDBG_FACILITY RPCDBG_TRANS 62c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#endif 63c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 649f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Leverstatic void rpcrdma_reset_frmrs(struct rpcrdma_ia *); 659f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever 66c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 67c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * internal functions 68c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 69c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 70c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 71c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * handle replies in tasklet context, using a single, global list 72c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * rdma tasklet function -- just turn around and call the func 73c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * for all replies on the list 74c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 75c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 76c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static DEFINE_SPINLOCK(rpcrdma_tk_lock_g); 77c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static LIST_HEAD(rpcrdma_tasklets_g); 78c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 79c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 80c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_run_tasklet(unsigned long data) 81c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 82c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep *rep; 83c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ void (*func)(struct rpcrdma_rep *); 84c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 85c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 86c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ data = data; 87c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&rpcrdma_tk_lock_g, flags); 88c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ while (!list_empty(&rpcrdma_tasklets_g)) { 89c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep = list_entry(rpcrdma_tasklets_g.next, 90c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep, rr_list); 91c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_del(&rep->rr_list); 92c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ func = rep->rr_func; 93c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_func = NULL; 94c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&rpcrdma_tk_lock_g, flags); 95c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 96c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (func) 97c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ func(rep); 98c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ else 99c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_recv_buffer_put(rep); 100c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 101c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&rpcrdma_tk_lock_g, flags); 102c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 103c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&rpcrdma_tk_lock_g, flags); 104c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 105c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 106c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static DECLARE_TASKLET(rpcrdma_tasklet_g, rpcrdma_run_tasklet, 0UL); 107c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 108c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 109c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_qp_async_error_upcall(struct ib_event *event, void *context) 110c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 111c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ep *ep = context; 112c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 113c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: QP error %X on device %s ep %p\n", 114c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, event->event, event->device->name, context); 115c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_connected == 1) { 116c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = -EIO; 117c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_func(ep); 118c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wake_up_all(&ep->rep_connect_wait); 119c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 120c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 121c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 122c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 123c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_cq_async_error_upcall(struct ib_event *event, void *context) 124c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 125c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ep *ep = context; 126c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 127c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: CQ error %X on device %s ep %p\n", 128c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, event->event, event->device->name, context); 129c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_connected == 1) { 130c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = -EIO; 131c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_func(ep); 132c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wake_up_all(&ep->rep_connect_wait); 133c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 134c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 135c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 136fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Leverstatic void 137fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Leverrpcrdma_sendcq_process_wc(struct ib_wc *wc) 138c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 139fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever struct rpcrdma_mw *frmr = (struct rpcrdma_mw *)(unsigned long)wc->wr_id; 140c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 141fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever dprintk("RPC: %s: frmr %p status %X opcode %d\n", 142fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever __func__, frmr, wc->status, wc->opcode); 143c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 144fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (wc->wr_id == 0ULL) 145c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return; 146dab7e3b8da5ef76143a7e609612c306898f8f8fcChuck Lever if (wc->status != IB_WC_SUCCESS) 1479f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever frmr->r.frmr.fr_state = FRMR_IS_STALE; 148c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 149c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 150fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Leverstatic int 1511c00dd0776543608e13c74a527660cb8cd28a74fChuck Leverrpcrdma_sendcq_poll(struct ib_cq *cq, struct rpcrdma_ep *ep) 152c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1531c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever struct ib_wc *wcs; 1548301a2c047cc25dabd645e5590c1db0ead4c5af4Chuck Lever int budget, count, rc; 155c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1568301a2c047cc25dabd645e5590c1db0ead4c5af4Chuck Lever budget = RPCRDMA_WC_BUDGET / RPCRDMA_POLLSIZE; 1571c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever do { 1581c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever wcs = ep->rep_send_wcs; 1591c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever 1601c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever rc = ib_poll_cq(cq, RPCRDMA_POLLSIZE, wcs); 1611c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever if (rc <= 0) 1621c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever return rc; 1631c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever 1641c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever count = rc; 1651c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever while (count-- > 0) 1661c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever rpcrdma_sendcq_process_wc(wcs++); 1678301a2c047cc25dabd645e5590c1db0ead4c5af4Chuck Lever } while (rc == RPCRDMA_POLLSIZE && --budget); 1681c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever return 0; 169fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever} 170c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 171fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever/* 172fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever * Handle send, fast_reg_mr, and local_inv completions. 173fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever * 174fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever * Send events are typically suppressed and thus do not result 175fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever * in an upcall. Occasionally one is signaled, however. This 176fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever * prevents the provider's completion queue from wrapping and 177fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever * losing a completion. 178fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever */ 179fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Leverstatic void 180fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Leverrpcrdma_sendcq_upcall(struct ib_cq *cq, void *cq_context) 181fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever{ 1821c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever struct rpcrdma_ep *ep = (struct rpcrdma_ep *)cq_context; 183fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever int rc; 184fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 1851c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever rc = rpcrdma_sendcq_poll(cq, ep); 186fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (rc) { 187fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever dprintk("RPC: %s: ib_poll_cq failed: %i\n", 188fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever __func__, rc); 189fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever return; 190c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 191c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1927f23f6f6e388d2003c4ecf5d558f3c2191e12530Chuck Lever rc = ib_req_notify_cq(cq, 1937f23f6f6e388d2003c4ecf5d558f3c2191e12530Chuck Lever IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS); 1947f23f6f6e388d2003c4ecf5d558f3c2191e12530Chuck Lever if (rc == 0) 1957f23f6f6e388d2003c4ecf5d558f3c2191e12530Chuck Lever return; 1967f23f6f6e388d2003c4ecf5d558f3c2191e12530Chuck Lever if (rc < 0) { 197fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever dprintk("RPC: %s: ib_req_notify_cq failed: %i\n", 198fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever __func__, rc); 199fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever return; 200fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever } 201fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 2021c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever rpcrdma_sendcq_poll(cq, ep); 203fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever} 204fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 205fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Leverstatic void 206bb96193d9104613cd87fb518f25db3fadc36432eChuck Leverrpcrdma_recvcq_process_wc(struct ib_wc *wc, struct list_head *sched_list) 207fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever{ 208fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever struct rpcrdma_rep *rep = 209fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever (struct rpcrdma_rep *)(unsigned long)wc->wr_id; 210fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 211fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever dprintk("RPC: %s: rep %p status %X opcode %X length %u\n", 212fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever __func__, rep, wc->status, wc->opcode, wc->byte_len); 213fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 214fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (wc->status != IB_WC_SUCCESS) { 215fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rep->rr_len = ~0U; 216fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever goto out_schedule; 217fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever } 218fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (wc->opcode != IB_WC_RECV) 219fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever return; 220fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 221fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rep->rr_len = wc->byte_len; 222fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever ib_dma_sync_single_for_cpu(rdmab_to_ia(rep->rr_buffer)->ri_id->device, 223fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rep->rr_iov.addr, rep->rr_len, DMA_FROM_DEVICE); 224fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 225fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (rep->rr_len >= 16) { 226fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever struct rpcrdma_msg *p = (struct rpcrdma_msg *)rep->rr_base; 227fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever unsigned int credits = ntohl(p->rm_credit); 228fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 229fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (credits == 0) 230fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever credits = 1; /* don't deadlock */ 231fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever else if (credits > rep->rr_buffer->rb_max_requests) 232fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever credits = rep->rr_buffer->rb_max_requests; 233fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever atomic_set(&rep->rr_buffer->rb_credits, credits); 234fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever } 235fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 236fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Leverout_schedule: 237bb96193d9104613cd87fb518f25db3fadc36432eChuck Lever list_add_tail(&rep->rr_list, sched_list); 238fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever} 239fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 240fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Leverstatic int 2411c00dd0776543608e13c74a527660cb8cd28a74fChuck Leverrpcrdma_recvcq_poll(struct ib_cq *cq, struct rpcrdma_ep *ep) 242fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever{ 243bb96193d9104613cd87fb518f25db3fadc36432eChuck Lever struct list_head sched_list; 2441c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever struct ib_wc *wcs; 2458301a2c047cc25dabd645e5590c1db0ead4c5af4Chuck Lever int budget, count, rc; 246bb96193d9104613cd87fb518f25db3fadc36432eChuck Lever unsigned long flags; 247fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 248bb96193d9104613cd87fb518f25db3fadc36432eChuck Lever INIT_LIST_HEAD(&sched_list); 2498301a2c047cc25dabd645e5590c1db0ead4c5af4Chuck Lever budget = RPCRDMA_WC_BUDGET / RPCRDMA_POLLSIZE; 2501c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever do { 2511c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever wcs = ep->rep_recv_wcs; 2521c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever 2531c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever rc = ib_poll_cq(cq, RPCRDMA_POLLSIZE, wcs); 2541c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever if (rc <= 0) 255bb96193d9104613cd87fb518f25db3fadc36432eChuck Lever goto out_schedule; 2561c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever 2571c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever count = rc; 2581c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever while (count-- > 0) 259bb96193d9104613cd87fb518f25db3fadc36432eChuck Lever rpcrdma_recvcq_process_wc(wcs++, &sched_list); 2608301a2c047cc25dabd645e5590c1db0ead4c5af4Chuck Lever } while (rc == RPCRDMA_POLLSIZE && --budget); 261bb96193d9104613cd87fb518f25db3fadc36432eChuck Lever rc = 0; 262bb96193d9104613cd87fb518f25db3fadc36432eChuck Lever 263bb96193d9104613cd87fb518f25db3fadc36432eChuck Leverout_schedule: 264bb96193d9104613cd87fb518f25db3fadc36432eChuck Lever spin_lock_irqsave(&rpcrdma_tk_lock_g, flags); 265bb96193d9104613cd87fb518f25db3fadc36432eChuck Lever list_splice_tail(&sched_list, &rpcrdma_tasklets_g); 266bb96193d9104613cd87fb518f25db3fadc36432eChuck Lever spin_unlock_irqrestore(&rpcrdma_tk_lock_g, flags); 267bb96193d9104613cd87fb518f25db3fadc36432eChuck Lever tasklet_schedule(&rpcrdma_tasklet_g); 268bb96193d9104613cd87fb518f25db3fadc36432eChuck Lever return rc; 269c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 270c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 271c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 272fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever * Handle receive completions. 273c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 274c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * It is reentrant but processes single events in order to maintain 275c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * ordering of receives to keep server credits. 276c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 277c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * It is the responsibility of the scheduled tasklet to return 278c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * recv buffers to the pool. NOTE: this affects synchronization of 279c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * connection shutdown. That is, the structures required for 280c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * the completion of the reply handler must remain intact until 281c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * all memory has been reclaimed. 282c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 283c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 284fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Leverrpcrdma_recvcq_upcall(struct ib_cq *cq, void *cq_context) 285c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 2861c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever struct rpcrdma_ep *ep = (struct rpcrdma_ep *)cq_context; 287c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 288c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 2891c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever rc = rpcrdma_recvcq_poll(cq, ep); 290fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (rc) { 291fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever dprintk("RPC: %s: ib_poll_cq failed: %i\n", 292fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever __func__, rc); 293c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return; 294fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever } 295c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 2967f23f6f6e388d2003c4ecf5d558f3c2191e12530Chuck Lever rc = ib_req_notify_cq(cq, 2977f23f6f6e388d2003c4ecf5d558f3c2191e12530Chuck Lever IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS); 2987f23f6f6e388d2003c4ecf5d558f3c2191e12530Chuck Lever if (rc == 0) 2997f23f6f6e388d2003c4ecf5d558f3c2191e12530Chuck Lever return; 3007f23f6f6e388d2003c4ecf5d558f3c2191e12530Chuck Lever if (rc < 0) { 301fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever dprintk("RPC: %s: ib_req_notify_cq failed: %i\n", 302c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 303c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return; 304c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 305c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 3061c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever rpcrdma_recvcq_poll(cq, ep); 307c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 308c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 309a7bc211ac926172ad20463afcf00ae7b9ebcd950Chuck Leverstatic void 310a7bc211ac926172ad20463afcf00ae7b9ebcd950Chuck Leverrpcrdma_flush_cqs(struct rpcrdma_ep *ep) 311a7bc211ac926172ad20463afcf00ae7b9ebcd950Chuck Lever{ 312a7bc211ac926172ad20463afcf00ae7b9ebcd950Chuck Lever rpcrdma_recvcq_upcall(ep->rep_attr.recv_cq, ep); 313a7bc211ac926172ad20463afcf00ae7b9ebcd950Chuck Lever rpcrdma_sendcq_upcall(ep->rep_attr.send_cq, ep); 314a7bc211ac926172ad20463afcf00ae7b9ebcd950Chuck Lever} 315a7bc211ac926172ad20463afcf00ae7b9ebcd950Chuck Lever 316c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#ifdef RPC_DEBUG 317c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static const char * const conn[] = { 318c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "address resolved", 319c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "address error", 320c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "route resolved", 321c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "route error", 322c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "connect request", 323c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "connect response", 324c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "connect error", 325c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "unreachable", 326c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "rejected", 327c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "established", 328c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "disconnected", 3298079fb785e34de6dff34bd846b8b79c212861edfChuck Lever "device removal", 3308079fb785e34de6dff34bd846b8b79c212861edfChuck Lever "multicast join", 3318079fb785e34de6dff34bd846b8b79c212861edfChuck Lever "multicast error", 3328079fb785e34de6dff34bd846b8b79c212861edfChuck Lever "address change", 3338079fb785e34de6dff34bd846b8b79c212861edfChuck Lever "timewait exit", 334c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\}; 3358079fb785e34de6dff34bd846b8b79c212861edfChuck Lever 3368079fb785e34de6dff34bd846b8b79c212861edfChuck Lever#define CONNECTION_MSG(status) \ 3378079fb785e34de6dff34bd846b8b79c212861edfChuck Lever ((status) < ARRAY_SIZE(conn) ? \ 3388079fb785e34de6dff34bd846b8b79c212861edfChuck Lever conn[(status)] : "unrecognized connection error") 339c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#endif 340c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 341c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static int 342c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_conn_upcall(struct rdma_cm_id *id, struct rdma_cm_event *event) 343c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 344c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_xprt *xprt = id->context; 345c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = &xprt->rx_ia; 346c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ep *ep = &xprt->rx_ep; 347ff0db0490a2eb6db4f03870987c5c8e3dd81e747Ingo Molnar#ifdef RPC_DEBUG 348c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct sockaddr_in *addr = (struct sockaddr_in *) &ep->rep_remote_addr; 349ff0db0490a2eb6db4f03870987c5c8e3dd81e747Ingo Molnar#endif 350c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_qp_attr attr; 351c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_qp_init_attr iattr; 352c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int connstate = 0; 353c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 354c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (event->event) { 355c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_ADDR_RESOLVED: 356c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_ROUTE_RESOLVED: 3575675add36e76b9487e7f9e689f854cb8d6afd9b4Tom Talpey ia->ri_async_rc = 0; 358c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ complete(&ia->ri_done); 359c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 360c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_ADDR_ERROR: 361c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_async_rc = -EHOSTUNREACH; 362c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: CM address resolution error, ep 0x%p\n", 363c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, ep); 364c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ complete(&ia->ri_done); 365c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 366c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_ROUTE_ERROR: 367c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_async_rc = -ENETUNREACH; 368c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: CM route resolution error, ep 0x%p\n", 369c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, ep); 370c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ complete(&ia->ri_done); 371c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 372c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_ESTABLISHED: 373c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = 1; 374c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_query_qp(ia->ri_id->qp, &attr, 375c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ IB_QP_MAX_QP_RD_ATOMIC | IB_QP_MAX_DEST_RD_ATOMIC, 376c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &iattr); 377c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: %d responder resources" 378c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " (%d initiator)\n", 379c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, attr.max_dest_rd_atomic, attr.max_rd_atomic); 380c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto connected; 381c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_CONNECT_ERROR: 382c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = -ENOTCONN; 383c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto connected; 384c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_UNREACHABLE: 385c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = -ENETDOWN; 386c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto connected; 387c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_REJECTED: 388c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = -ECONNREFUSED; 389c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto connected; 390c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_DISCONNECTED: 391c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = -ECONNABORTED; 392c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto connected; 393c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_DEVICE_REMOVAL: 394c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = -ENODEV; 395c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\connected: 396c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ atomic_set(&rpcx_to_rdmax(ep->rep_xprt)->rx_buf.rb_credits, 1); 397c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: %sconnected\n", 398c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, connstate > 0 ? "" : "dis"); 399c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = connstate; 400c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_func(ep); 401c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wake_up_all(&ep->rep_connect_wait); 4028079fb785e34de6dff34bd846b8b79c212861edfChuck Lever /*FALLTHROUGH*/ 403c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 4048079fb785e34de6dff34bd846b8b79c212861edfChuck Lever dprintk("RPC: %s: %pI4:%u (ep 0x%p): %s\n", 4058079fb785e34de6dff34bd846b8b79c212861edfChuck Lever __func__, &addr->sin_addr.s_addr, 4068079fb785e34de6dff34bd846b8b79c212861edfChuck Lever ntohs(addr->sin_port), ep, 4078079fb785e34de6dff34bd846b8b79c212861edfChuck Lever CONNECTION_MSG(event->event)); 408c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 409c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 410c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 411b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey#ifdef RPC_DEBUG 412b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey if (connstate == 1) { 413b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey int ird = attr.max_dest_rd_atomic; 414b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey int tird = ep->rep_remote_cma.responder_resources; 41521454aaad30651ba0dcc16fe5271bc12ee21f132Harvey Harrison printk(KERN_INFO "rpcrdma: connection to %pI4:%u " 416b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey "on %s, memreg %d slots %d ird %d%s\n", 41721454aaad30651ba0dcc16fe5271bc12ee21f132Harvey Harrison &addr->sin_addr.s_addr, 418b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey ntohs(addr->sin_port), 419b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey ia->ri_id->device->name, 420b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey ia->ri_memreg_strategy, 421b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey xprt->rx_buf.rb_max_requests, 422b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey ird, ird < 4 && ird < tird / 2 ? " (low!)" : ""); 423b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey } else if (connstate < 0) { 42421454aaad30651ba0dcc16fe5271bc12ee21f132Harvey Harrison printk(KERN_INFO "rpcrdma: connection to %pI4:%u closed (%d)\n", 42521454aaad30651ba0dcc16fe5271bc12ee21f132Harvey Harrison &addr->sin_addr.s_addr, 426b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey ntohs(addr->sin_port), 427b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey connstate); 428b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey } 429b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey#endif 430b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey 431c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 432c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 433c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 434c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static struct rdma_cm_id * 435c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_create_id(struct rpcrdma_xprt *xprt, 436c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia, struct sockaddr *addr) 437c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 438c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rdma_cm_id *id; 439c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 440c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 4411a954051b0cf79bd67e5f9db40333e3a9b1d05d2Tom Talpey init_completion(&ia->ri_done); 4421a954051b0cf79bd67e5f9db40333e3a9b1d05d2Tom Talpey 443b26f9b9949013fec31b23c426fc463164ae08891Sean Hefty id = rdma_create_id(rpcrdma_conn_upcall, xprt, RDMA_PS_TCP, IB_QPT_RC); 444c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(id)) { 445c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(id); 446c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_create_id() failed %i\n", 447c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 448c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return id; 449c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 450c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 4515675add36e76b9487e7f9e689f854cb8d6afd9b4Tom Talpey ia->ri_async_rc = -ETIMEDOUT; 452c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rdma_resolve_addr(id, NULL, addr, RDMA_RESOLVE_TIMEOUT); 453c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 454c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_resolve_addr() failed %i\n", 455c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 456c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 457c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 4585675add36e76b9487e7f9e689f854cb8d6afd9b4Tom Talpey wait_for_completion_interruptible_timeout(&ia->ri_done, 4595675add36e76b9487e7f9e689f854cb8d6afd9b4Tom Talpey msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1); 460c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ia->ri_async_rc; 461c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 462c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 463c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 4645675add36e76b9487e7f9e689f854cb8d6afd9b4Tom Talpey ia->ri_async_rc = -ETIMEDOUT; 465c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rdma_resolve_route(id, RDMA_RESOLVE_TIMEOUT); 466c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 467c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_resolve_route() failed %i\n", 468c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 469c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 470c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 4715675add36e76b9487e7f9e689f854cb8d6afd9b4Tom Talpey wait_for_completion_interruptible_timeout(&ia->ri_done, 4725675add36e76b9487e7f9e689f854cb8d6afd9b4Tom Talpey msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1); 473c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ia->ri_async_rc; 474c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 475c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 476c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 477c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return id; 478c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 479c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out: 480c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rdma_destroy_id(id); 481c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return ERR_PTR(rc); 482c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 483c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 484c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 485c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Drain any cq, prior to teardown. 486c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 487c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 488c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_clean_cq(struct ib_cq *cq) 489c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 490c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_wc wc; 491c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int count = 0; 492c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 493c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ while (1 == ib_poll_cq(cq, 1, &wc)) 494c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ++count; 495c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 496c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (count) 497c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: flushed %d events (last 0x%x)\n", 498c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, count, wc.opcode); 499c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 500c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 501c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 502c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Exported functions. 503c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 504c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 505c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 506c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Open and initialize an Interface Adapter. 507c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o initializes fields of struct rpcrdma_ia, including 508c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * interface and provider attributes and protection zone. 509c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 510c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 511c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ia_open(struct rpcrdma_xprt *xprt, struct sockaddr *addr, int memreg) 512c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 513bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey int rc, mem_priv; 514bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey struct ib_device_attr devattr; 515c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = &xprt->rx_ia; 516c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 517c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_id = rpcrdma_create_id(xprt, ia, addr); 518c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(ia->ri_id)) { 519c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(ia->ri_id); 520c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out1; 521c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 522c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 523c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_pd = ib_alloc_pd(ia->ri_id->device); 524c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(ia->ri_pd)) { 525c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(ia->ri_pd); 526c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_alloc_pd() failed %i\n", 527c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 528c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out2; 529c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 530c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 531c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 532bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey * Query the device to determine if the requested memory 533bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey * registration strategy is supported. If it isn't, set the 534bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey * strategy to a globally supported model. 535bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey */ 536bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey rc = ib_query_device(ia->ri_id->device, &devattr); 537bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey if (rc) { 538bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey dprintk("RPC: %s: ib_query_device failed %d\n", 539bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey __func__, rc); 540bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey goto out2; 541bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey } 542bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey 543bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey if (devattr.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY) { 544bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey ia->ri_have_dma_lkey = 1; 545bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey ia->ri_dma_lkey = ia->ri_id->device->local_dma_lkey; 546bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey } 547bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey 548f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever if (memreg == RPCRDMA_FRMR) { 5493197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey /* Requires both frmr reg and local dma lkey */ 5503197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey if ((devattr.device_cap_flags & 5513197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey (IB_DEVICE_MEM_MGT_EXTENSIONS|IB_DEVICE_LOCAL_DMA_LKEY)) != 5523197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey (IB_DEVICE_MEM_MGT_EXTENSIONS|IB_DEVICE_LOCAL_DMA_LKEY)) { 5533197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey dprintk("RPC: %s: FRMR registration " 554f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever "not supported by HCA\n", __func__); 555f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever memreg = RPCRDMA_MTHCAFMR; 5560fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise } else { 5570fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise /* Mind the ia limit on FRMR page list depth */ 5580fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise ia->ri_max_frmr_depth = min_t(unsigned int, 5590fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise RPCRDMA_MAX_DATA_SEGS, 5600fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise devattr.max_fast_reg_page_list_len); 5613197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey } 562f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever } 563f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever if (memreg == RPCRDMA_MTHCAFMR) { 564f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever if (!ia->ri_id->device->alloc_fmr) { 565f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever dprintk("RPC: %s: MTHCAFMR registration " 566f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever "not supported by HCA\n", __func__); 567f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever memreg = RPCRDMA_ALLPHYSICAL; 568f10eafd3a6ce9da7e96999c124b643ea6c4921f3Chuck Lever } 569bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey } 570bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey 571bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey /* 572c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Optionally obtain an underlying physical identity mapping in 573c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * order to do a memory window-based bind. This base registration 574c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * is protected from remote access - that is enabled only by binding 575c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * for the specific bytes targeted during each RPC operation, and 576c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * revoked after the corresponding completion similar to a storage 577c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * adapter. 578c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 579bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey switch (memreg) { 5803197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey case RPCRDMA_FRMR: 581bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey break; 582bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_ALLPHYSICAL: 583bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey mem_priv = IB_ACCESS_LOCAL_WRITE | 584bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey IB_ACCESS_REMOTE_WRITE | 585bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey IB_ACCESS_REMOTE_READ; 586bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey goto register_setup; 587bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_MTHCAFMR: 588bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey if (ia->ri_have_dma_lkey) 589c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 590bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey mem_priv = IB_ACCESS_LOCAL_WRITE; 591bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey register_setup: 592c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_bind_mem = ib_get_dma_mr(ia->ri_pd, mem_priv); 593c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(ia->ri_bind_mem)) { 594c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ printk(KERN_ALERT "%s: ib_get_dma_mr for " 5950ac531c1832318efa3dc3d723e356a7e09330e80Chuck Lever "phys register failed with %lX\n", 596c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, PTR_ERR(ia->ri_bind_mem)); 5970ac531c1832318efa3dc3d723e356a7e09330e80Chuck Lever rc = -ENOMEM; 5980ac531c1832318efa3dc3d723e356a7e09330e80Chuck Lever goto out2; 599c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 600bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey break; 601bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey default: 602cdd9ade711599e7672a635add0406080856f8b92Chuck Lever printk(KERN_ERR "RPC: Unsupported memory " 603cdd9ade711599e7672a635add0406080856f8b92Chuck Lever "registration mode: %d\n", memreg); 604cdd9ade711599e7672a635add0406080856f8b92Chuck Lever rc = -ENOMEM; 605bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey goto out2; 606c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 607bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey dprintk("RPC: %s: memory registration strategy is %d\n", 608bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey __func__, memreg); 609c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 610c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Else will do memory reg/dereg for each chunk */ 611c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_memreg_strategy = memreg; 612c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 61373806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever rwlock_init(&ia->ri_qplock); 614c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 615c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out2: 616c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rdma_destroy_id(ia->ri_id); 617fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey ia->ri_id = NULL; 618c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out1: 619c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 620c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 621c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 622c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 623c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Clean up/close an IA. 624c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o if event handles and PD have been initialized, free them. 625c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o close the IA 626c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 627c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void 628c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ia_close(struct rpcrdma_ia *ia) 629c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 630c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 631c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 632c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: entering\n", __func__); 633c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ia->ri_bind_mem != NULL) { 634c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_dereg_mr(ia->ri_bind_mem); 635c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_dereg_mr returned %i\n", 636c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 637c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 638fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey if (ia->ri_id != NULL && !IS_ERR(ia->ri_id)) { 639fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey if (ia->ri_id->qp) 640fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey rdma_destroy_qp(ia->ri_id); 641fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey rdma_destroy_id(ia->ri_id); 642fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey ia->ri_id = NULL; 643fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey } 644c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ia->ri_pd != NULL && !IS_ERR(ia->ri_pd)) { 645c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_dealloc_pd(ia->ri_pd); 646c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_dealloc_pd returned %i\n", 647c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 648c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 649c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 650c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 651c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 652c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Create unconnected endpoint. 653c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 654c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 655c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia, 656c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_create_data_internal *cdata) 657c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 658c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_device_attr devattr; 659fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever struct ib_cq *sendcq, *recvcq; 6605d40a8a525c8165bafed233cf0f137e8d10d7e92Chuck Lever int rc, err; 661c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 662c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_query_device(ia->ri_id->device, &devattr); 663c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 664c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_query_device failed %d\n", 665c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 666c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 667c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 668c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 669c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* check provider's send/recv wr limits */ 670c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (cdata->max_requests > devattr.max_qp_wr) 671c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ cdata->max_requests = devattr.max_qp_wr; 672c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 673c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.event_handler = rpcrdma_qp_async_error_upcall; 674c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.qp_context = ep; 675c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* send_cq and recv_cq initialized below */ 676c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.srq = NULL; 677c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_wr = cdata->max_requests; 678c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 6790fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise case RPCRDMA_FRMR: { 6800fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise int depth = 7; 6810fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise 68215cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker /* Add room for frmr register and invalidate WRs. 68315cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker * 1. FRMR reg WR for head 68415cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker * 2. FRMR invalidate WR for head 6850fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise * 3. N FRMR reg WRs for pagelist 6860fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise * 4. N FRMR invalidate WRs for pagelist 68715cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker * 5. FRMR reg WR for tail 68815cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker * 6. FRMR invalidate WR for tail 68915cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker * 7. The RDMA_SEND WR 69015cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker */ 6910fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise 6920fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise /* Calculate N if the device max FRMR depth is smaller than 6930fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise * RPCRDMA_MAX_DATA_SEGS. 6940fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise */ 6950fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise if (ia->ri_max_frmr_depth < RPCRDMA_MAX_DATA_SEGS) { 6960fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise int delta = RPCRDMA_MAX_DATA_SEGS - 6970fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise ia->ri_max_frmr_depth; 6980fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise 6990fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise do { 7000fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise depth += 2; /* FRMR reg + invalidate */ 7010fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise delta -= ia->ri_max_frmr_depth; 7020fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise } while (delta > 0); 7030fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise 7040fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise } 7050fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise ep->rep_attr.cap.max_send_wr *= depth; 70615cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker if (ep->rep_attr.cap.max_send_wr > devattr.max_qp_wr) { 7070fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise cdata->max_requests = devattr.max_qp_wr / depth; 70815cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker if (!cdata->max_requests) 70915cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker return -EINVAL; 7100fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise ep->rep_attr.cap.max_send_wr = cdata->max_requests * 7110fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise depth; 71215cdc644b268a9a9ce73ce0b153129222c254b7bTom Tucker } 7133197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey break; 7140fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise } 715c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 716c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 717c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 718c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_recv_wr = cdata->max_requests; 719c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_sge = (cdata->padding ? 4 : 2); 720c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_recv_sge = 1; 721c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_inline_data = 0; 722c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.sq_sig_type = IB_SIGNAL_REQ_WR; 723c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.qp_type = IB_QPT_RC; 724c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.port_num = ~0; 725c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 726c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: requested max: dtos: send %d recv %d; " 727c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "iovs: send %d recv %d\n", 728c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, 729c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_wr, 730c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_recv_wr, 731c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_sge, 732c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_recv_sge); 733c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 734c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* set trigger for requesting send completion */ 735fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever ep->rep_cqinit = ep->rep_attr.cap.max_send_wr/2 - 1; 736c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_cqinit <= 2) 737c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_cqinit = 0; 738c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ INIT_CQCOUNT(ep); 739c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_ia = ia; 740c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ init_waitqueue_head(&ep->rep_connect_wait); 741254f91e2fa1f4cc18fd2eb9d5481888ffe126d5bChuck Lever INIT_DELAYED_WORK(&ep->rep_connect_worker, rpcrdma_connect_worker); 742c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 743fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever sendcq = ib_create_cq(ia->ri_id->device, rpcrdma_sendcq_upcall, 7441c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever rpcrdma_cq_async_error_upcall, ep, 745c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_wr + 1, 0); 746fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (IS_ERR(sendcq)) { 747fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rc = PTR_ERR(sendcq); 748fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever dprintk("RPC: %s: failed to create send CQ: %i\n", 749c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 750c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out1; 751c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 752c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 753fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rc = ib_req_notify_cq(sendcq, IB_CQ_NEXT_COMP); 754c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 755c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_req_notify_cq failed: %i\n", 756c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 757c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out2; 758c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 759c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 760fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever recvcq = ib_create_cq(ia->ri_id->device, rpcrdma_recvcq_upcall, 7611c00dd0776543608e13c74a527660cb8cd28a74fChuck Lever rpcrdma_cq_async_error_upcall, ep, 762fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever ep->rep_attr.cap.max_recv_wr + 1, 0); 763fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (IS_ERR(recvcq)) { 764fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rc = PTR_ERR(recvcq); 765fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever dprintk("RPC: %s: failed to create recv CQ: %i\n", 766fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever __func__, rc); 767fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever goto out2; 768fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever } 769fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 770fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rc = ib_req_notify_cq(recvcq, IB_CQ_NEXT_COMP); 771fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (rc) { 772fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever dprintk("RPC: %s: ib_req_notify_cq failed: %i\n", 773fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever __func__, rc); 774fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever ib_destroy_cq(recvcq); 775fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever goto out2; 776fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever } 777fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 778fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever ep->rep_attr.send_cq = sendcq; 779fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever ep->rep_attr.recv_cq = recvcq; 780c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 781c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Initialize cma parameters */ 782c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 783c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* RPC/RDMA does not use private data */ 784c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.private_data = NULL; 785c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.private_data_len = 0; 786c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 787c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Client offers RDMA Read but does not initiate */ 788b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker ep->rep_remote_cma.initiator_depth = 0; 78903ff8821eb5ed168792667cfc3ddff903e97af99Chuck Lever if (devattr.max_qp_rd_atom > 32) /* arbitrary but <= 255 */ 790b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker ep->rep_remote_cma.responder_resources = 32; 791b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker else 792c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.responder_resources = devattr.max_qp_rd_atom; 793c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 794c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.retry_count = 7; 795c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.flow_control = 0; 796c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.rnr_retry_count = 0; 797c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 798c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 799c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 800c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out2: 801fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever err = ib_destroy_cq(sendcq); 8025d40a8a525c8165bafed233cf0f137e8d10d7e92Chuck Lever if (err) 8035d40a8a525c8165bafed233cf0f137e8d10d7e92Chuck Lever dprintk("RPC: %s: ib_destroy_cq returned %i\n", 8045d40a8a525c8165bafed233cf0f137e8d10d7e92Chuck Lever __func__, err); 805c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out1: 806c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 807c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 808c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 809c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 810c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * rpcrdma_ep_destroy 811c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 812c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Disconnect and destroy endpoint. After this, the only 813c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * valid operations on the ep are to free it (if dynamically 814c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * allocated) or re-create it. 815c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 8167f1d54191ed6fa0f79f584fe3ebf6519738e817fChuck Levervoid 817c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_destroy(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) 818c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 819c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 820c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 821c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: entering, connected is %d\n", 822c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, ep->rep_connected); 823c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 824254f91e2fa1f4cc18fd2eb9d5481888ffe126d5bChuck Lever cancel_delayed_work_sync(&ep->rep_connect_worker); 825254f91e2fa1f4cc18fd2eb9d5481888ffe126d5bChuck Lever 826c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ia->ri_id->qp) { 827282191cb725db9a1aa80269e8369b06e9270a948Chuck Lever rpcrdma_ep_disconnect(ep, ia); 828fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey rdma_destroy_qp(ia->ri_id); 829fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey ia->ri_id->qp = NULL; 830c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 831c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 832c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* padding - could be done in rpcrdma_buffer_destroy... */ 833c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_pad_mr) { 834c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_deregister_internal(ia, ep->rep_pad_mr, &ep->rep_pad); 835c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_pad_mr = NULL; 836c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 837c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 838fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rpcrdma_clean_cq(ep->rep_attr.recv_cq); 839fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rc = ib_destroy_cq(ep->rep_attr.recv_cq); 840fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever if (rc) 841fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever dprintk("RPC: %s: ib_destroy_cq returned %i\n", 842fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever __func__, rc); 843fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever 844fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rpcrdma_clean_cq(ep->rep_attr.send_cq); 845fc66448549bbb77f2f1a38b270ab2d6b6a22da33Chuck Lever rc = ib_destroy_cq(ep->rep_attr.send_cq); 846c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 847c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_destroy_cq returned %i\n", 848c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 849c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 850c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 851c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 852c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Connect unconnected endpoint. 853c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 854c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 855c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) 856c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 85773806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever struct rdma_cm_id *id, *old; 858c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc = 0; 859c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int retry_count = 0; 860c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 861c055551e97e1ca00781bc41523f829e05a8afed7Tom Talpey if (ep->rep_connected != 0) { 862c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_xprt *xprt; 863c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\retry: 864ec62f40d3505a643497d105c297093bb90afd44eChuck Lever dprintk("RPC: %s: reconnecting...\n", __func__); 865282191cb725db9a1aa80269e8369b06e9270a948Chuck Lever 866282191cb725db9a1aa80269e8369b06e9270a948Chuck Lever rpcrdma_ep_disconnect(ep, ia); 867a7bc211ac926172ad20463afcf00ae7b9ebcd950Chuck Lever rpcrdma_flush_cqs(ep); 868c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 8699f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever if (ia->ri_memreg_strategy == RPCRDMA_FRMR) 8709f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever rpcrdma_reset_frmrs(ia); 8719f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever 872c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ xprt = container_of(ia, struct rpcrdma_xprt, rx_ia); 873c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ id = rpcrdma_create_id(xprt, ia, 874c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (struct sockaddr *)&xprt->rx_data.addr); 875c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(id)) { 876ec62f40d3505a643497d105c297093bb90afd44eChuck Lever rc = -EHOSTUNREACH; 877c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 878c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 879c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* TEMP TEMP TEMP - fail if new device: 880c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Deregister/remarshal *all* requests! 881c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Close and recreate adapter, pd, etc! 882c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Re-determine all attributes still sane! 883c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * More stuff I haven't thought of! 884c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Rrrgh! 885c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 886c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ia->ri_id->device != id->device) { 887c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ printk("RPC: %s: can't reconnect on " 888c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "different device!\n", __func__); 889c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rdma_destroy_id(id); 890ec62f40d3505a643497d105c297093bb90afd44eChuck Lever rc = -ENETUNREACH; 891c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 892c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 893c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* END TEMP */ 894ec62f40d3505a643497d105c297093bb90afd44eChuck Lever rc = rdma_create_qp(id, ia->ri_pd, &ep->rep_attr); 895ec62f40d3505a643497d105c297093bb90afd44eChuck Lever if (rc) { 896ec62f40d3505a643497d105c297093bb90afd44eChuck Lever dprintk("RPC: %s: rdma_create_qp failed %i\n", 897ec62f40d3505a643497d105c297093bb90afd44eChuck Lever __func__, rc); 898ec62f40d3505a643497d105c297093bb90afd44eChuck Lever rdma_destroy_id(id); 899ec62f40d3505a643497d105c297093bb90afd44eChuck Lever rc = -ENETUNREACH; 900ec62f40d3505a643497d105c297093bb90afd44eChuck Lever goto out; 901ec62f40d3505a643497d105c297093bb90afd44eChuck Lever } 90273806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever 90373806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever write_lock(&ia->ri_qplock); 90473806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever old = ia->ri_id; 905c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_id = id; 90673806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever write_unlock(&ia->ri_qplock); 90773806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever 90873806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever rdma_destroy_qp(old); 90973806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever rdma_destroy_id(old); 910ec62f40d3505a643497d105c297093bb90afd44eChuck Lever } else { 911ec62f40d3505a643497d105c297093bb90afd44eChuck Lever dprintk("RPC: %s: connecting...\n", __func__); 912ec62f40d3505a643497d105c297093bb90afd44eChuck Lever rc = rdma_create_qp(ia->ri_id, ia->ri_pd, &ep->rep_attr); 913ec62f40d3505a643497d105c297093bb90afd44eChuck Lever if (rc) { 914ec62f40d3505a643497d105c297093bb90afd44eChuck Lever dprintk("RPC: %s: rdma_create_qp failed %i\n", 915ec62f40d3505a643497d105c297093bb90afd44eChuck Lever __func__, rc); 916ec62f40d3505a643497d105c297093bb90afd44eChuck Lever /* do not update ep->rep_connected */ 917ec62f40d3505a643497d105c297093bb90afd44eChuck Lever return -ENETUNREACH; 918ec62f40d3505a643497d105c297093bb90afd44eChuck Lever } 919c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 920c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 921c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = 0; 922c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 923c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rdma_connect(ia->ri_id, &ep->rep_remote_cma); 924c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 925c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_connect() failed with %i\n", 926c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 927c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 928c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 929c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 930c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wait_event_interruptible(ep->rep_connect_wait, ep->rep_connected != 0); 931c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 932c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 933c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Check state. A non-peer reject indicates no listener 934c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * (ECONNREFUSED), which may be a transient state. All 935c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * others indicate a transport condition which has already 936c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * undergone a best-effort. 937c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 938f64f9e719261a87818dd192a3a2352e5b20fbd0fJoe Perches if (ep->rep_connected == -ECONNREFUSED && 939f64f9e719261a87818dd192a3a2352e5b20fbd0fJoe Perches ++retry_count <= RDMA_CONNECT_RETRY_MAX) { 940c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: non-peer_reject, retry\n", __func__); 941c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto retry; 942c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 943c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_connected <= 0) { 944c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Sometimes, the only way to reliably connect to remote 945c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * CMs is to use same nonzero values for ORD and IRD. */ 946b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker if (retry_count++ <= RDMA_CONNECT_RETRY_MAX + 1 && 947b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker (ep->rep_remote_cma.responder_resources == 0 || 948b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker ep->rep_remote_cma.initiator_depth != 949b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker ep->rep_remote_cma.responder_resources)) { 950b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker if (ep->rep_remote_cma.responder_resources == 0) 951b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker ep->rep_remote_cma.responder_resources = 1; 952b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker ep->rep_remote_cma.initiator_depth = 953b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker ep->rep_remote_cma.responder_resources; 954c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto retry; 955b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker } 956c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ep->rep_connected; 957c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } else { 958c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: connected\n", __func__); 959c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 960c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 961c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out: 962c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 963c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = rc; 964c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 965c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 966c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 967c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 968c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * rpcrdma_ep_disconnect 969c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 970c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * This is separate from destroy to facilitate the ability 971c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * to reconnect without recreating the endpoint. 972c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 973c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * This call is not reentrant, and must not be made in parallel 974c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * on the same endpoint. 975c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 976282191cb725db9a1aa80269e8369b06e9270a948Chuck Levervoid 977c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_disconnect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) 978c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 979c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 980c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 981a7bc211ac926172ad20463afcf00ae7b9ebcd950Chuck Lever rpcrdma_flush_cqs(ep); 982c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rdma_disconnect(ia->ri_id); 983c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (!rc) { 984c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* returns without wait if not connected */ 985c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wait_event_interruptible(ep->rep_connect_wait, 986c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected != 1); 987c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: after wait, %sconnected\n", __func__, 988c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (ep->rep_connected == 1) ? "still " : "dis"); 989c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } else { 990c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_disconnect %i\n", __func__, rc); 991c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = rc; 992c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 993c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 994c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 9952e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Leverstatic int 9962e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Leverrpcrdma_init_fmrs(struct rpcrdma_ia *ia, struct rpcrdma_buffer *buf) 9972e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever{ 9982e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever int mr_access_flags = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ; 9992e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever struct ib_fmr_attr fmr_attr = { 10002e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever .max_pages = RPCRDMA_MAX_DATA_SEGS, 10012e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever .max_maps = 1, 10022e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever .page_shift = PAGE_SHIFT 10032e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever }; 10042e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever struct rpcrdma_mw *r; 10052e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever int i, rc; 10062e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 10072e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever i = (buf->rb_max_requests + 1) * RPCRDMA_MAX_SEGS; 10082e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever dprintk("RPC: %s: initalizing %d FMRs\n", __func__, i); 10092e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 10102e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever while (i--) { 10112e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever r = kzalloc(sizeof(*r), GFP_KERNEL); 10122e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever if (r == NULL) 10132e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever return -ENOMEM; 10142e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 10152e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever r->r.fmr = ib_alloc_fmr(ia->ri_pd, mr_access_flags, &fmr_attr); 10162e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever if (IS_ERR(r->r.fmr)) { 10172e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever rc = PTR_ERR(r->r.fmr); 10182e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever dprintk("RPC: %s: ib_alloc_fmr failed %i\n", 10192e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever __func__, rc); 10202e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever goto out_free; 10212e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever } 10222e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 10232e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever list_add(&r->mw_list, &buf->rb_mws); 10242e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever list_add(&r->mw_all, &buf->rb_all); 10252e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever } 10262e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever return 0; 10272e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 10282e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Leverout_free: 10292e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever kfree(r); 10302e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever return rc; 10312e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever} 10322e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 10332e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Leverstatic int 10342e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Leverrpcrdma_init_frmrs(struct rpcrdma_ia *ia, struct rpcrdma_buffer *buf) 10352e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever{ 10362e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever struct rpcrdma_frmr *f; 10372e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever struct rpcrdma_mw *r; 10382e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever int i, rc; 10392e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 10402e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever i = (buf->rb_max_requests + 1) * RPCRDMA_MAX_SEGS; 10412e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever dprintk("RPC: %s: initalizing %d FRMRs\n", __func__, i); 10422e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 10432e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever while (i--) { 10442e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever r = kzalloc(sizeof(*r), GFP_KERNEL); 10452e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever if (r == NULL) 10462e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever return -ENOMEM; 10472e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever f = &r->r.frmr; 10482e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 10492e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever f->fr_mr = ib_alloc_fast_reg_mr(ia->ri_pd, 10502e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever ia->ri_max_frmr_depth); 10512e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever if (IS_ERR(f->fr_mr)) { 10522e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever rc = PTR_ERR(f->fr_mr); 10532e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever dprintk("RPC: %s: ib_alloc_fast_reg_mr " 10542e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever "failed %i\n", __func__, rc); 10552e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever goto out_free; 10562e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever } 10572e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 10582e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever f->fr_pgl = ib_alloc_fast_reg_page_list(ia->ri_id->device, 10592e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever ia->ri_max_frmr_depth); 10602e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever if (IS_ERR(f->fr_pgl)) { 10612e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever rc = PTR_ERR(f->fr_pgl); 10622e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever dprintk("RPC: %s: ib_alloc_fast_reg_page_list " 10632e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever "failed %i\n", __func__, rc); 10642e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 10652e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever ib_dereg_mr(f->fr_mr); 10662e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever goto out_free; 10672e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever } 10682e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 10692e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever list_add(&r->mw_list, &buf->rb_mws); 10702e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever list_add(&r->mw_all, &buf->rb_all); 10712e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever } 10722e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 10732e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever return 0; 10742e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 10752e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Leverout_free: 10762e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever kfree(r); 10772e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever return rc; 10782e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever} 10792e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 1080c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1081c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_buffer_create(struct rpcrdma_buffer *buf, struct rpcrdma_ep *ep, 1082c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia, struct rpcrdma_create_data_internal *cdata) 1083c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1084c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ char *p; 108565866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever size_t len, rlen, wlen; 1086c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int i, rc; 1087c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1088c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_max_requests = cdata->max_requests; 1089c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_init(&buf->rb_lock); 1090c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ atomic_set(&buf->rb_credits, 1); 1091c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1092c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Need to allocate: 1093c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 1. arrays for send and recv pointers 1094c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 2. arrays of struct rpcrdma_req to fill in pointers 1095c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 3. array of struct rpcrdma_rep for replies 1096c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 4. padding, if any 1097c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Send/recv buffers in req/rep need to be registered 1098c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1099c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len = buf->rb_max_requests * 1100c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (sizeof(struct rpcrdma_req *) + sizeof(struct rpcrdma_rep *)); 1101c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len += cdata->padding; 1102c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1103c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ p = kzalloc(len, GFP_KERNEL); 1104c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (p == NULL) { 1105c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: req_t/rep_t/pad kzalloc(%zd) failed\n", 1106c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, len); 1107c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = -ENOMEM; 1108c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1109c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1110c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_pool = p; /* for freeing it later */ 1111c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1112c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_send_bufs = (struct rpcrdma_req **) p; 1113c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ p = (char *) &buf->rb_send_bufs[buf->rb_max_requests]; 1114c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_recv_bufs = (struct rpcrdma_rep **) p; 1115c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ p = (char *) &buf->rb_recv_bufs[buf->rb_max_requests]; 1116c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1117c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 1118c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Register the zeroed pad buffer, if any. 1119c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1120c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (cdata->padding) { 1121c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_register_internal(ia, p, cdata->padding, 1122c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &ep->rep_pad_mr, &ep->rep_pad); 1123c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1124c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1125c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1126c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ p += cdata->padding; 1127c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1128c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ INIT_LIST_HEAD(&buf->rb_mws); 11293111d72c7ced444b1034f6e365e0e02444c68aa8Chuck Lever INIT_LIST_HEAD(&buf->rb_all); 1130c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 11313197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey case RPCRDMA_FRMR: 11322e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever rc = rpcrdma_init_frmrs(ia, buf); 11332e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever if (rc) 11342e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever goto out; 11353197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey break; 1136c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 11372e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever rc = rpcrdma_init_fmrs(ia, buf); 11382e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever if (rc) 11392e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever goto out; 1140c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1141c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 1142c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1143c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1144c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1145c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 1146c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Allocate/init the request/reply buffers. Doing this 1147c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * using kmalloc for now -- one for each buf. 1148c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 114965866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever wlen = 1 << fls(cdata->inline_wsize + sizeof(struct rpcrdma_req)); 115065866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever rlen = 1 << fls(cdata->inline_rsize + sizeof(struct rpcrdma_rep)); 115165866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever dprintk("RPC: %s: wlen = %zu, rlen = %zu\n", 115265866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever __func__, wlen, rlen); 115365866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever 1154c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ for (i = 0; i < buf->rb_max_requests; i++) { 1155c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_req *req; 1156c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep *rep; 1157c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 115865866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever req = kmalloc(wlen, GFP_KERNEL); 1159c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (req == NULL) { 1160c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: request buffer %d alloc" 1161c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " failed\n", __func__, i); 1162c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = -ENOMEM; 1163c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1164c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1165c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ memset(req, 0, sizeof(struct rpcrdma_req)); 1166c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_send_bufs[i] = req; 1167c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_send_bufs[i]->rl_buffer = buf; 1168c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1169c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_register_internal(ia, req->rl_base, 117065866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever wlen - offsetof(struct rpcrdma_req, rl_base), 1171c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_send_bufs[i]->rl_handle, 1172c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_send_bufs[i]->rl_iov); 1173c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1174c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1175c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 117665866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever buf->rb_send_bufs[i]->rl_size = wlen - 117765866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever sizeof(struct rpcrdma_req); 1178c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 117965866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever rep = kmalloc(rlen, GFP_KERNEL); 1180c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rep == NULL) { 1181c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: reply buffer %d alloc failed\n", 1182c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, i); 1183c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = -ENOMEM; 1184c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1185c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1186c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ memset(rep, 0, sizeof(struct rpcrdma_rep)); 1187c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_recv_bufs[i] = rep; 1188c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_recv_bufs[i]->rr_buffer = buf; 1189c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1190c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_register_internal(ia, rep->rr_base, 119165866f8259851cea5e356d2fd46fc37a4e26330eChuck Lever rlen - offsetof(struct rpcrdma_rep, rr_base), 1192c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_recv_bufs[i]->rr_handle, 1193c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_recv_bufs[i]->rr_iov); 1194c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1195c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1196c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1197c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1198c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: max_requests %d\n", 1199c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, buf->rb_max_requests); 1200c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* done */ 1201c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 1202c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out: 1203c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_buffer_destroy(buf); 1204c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 1205c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1206c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 12072e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Leverstatic void 12082e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Leverrpcrdma_destroy_fmrs(struct rpcrdma_buffer *buf) 12092e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever{ 12102e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever struct rpcrdma_mw *r; 12112e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever int rc; 12122e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 12132e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever while (!list_empty(&buf->rb_all)) { 12142e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever r = list_entry(buf->rb_all.next, struct rpcrdma_mw, mw_all); 12152e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever list_del(&r->mw_all); 12162e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever list_del(&r->mw_list); 12172e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 12182e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever rc = ib_dealloc_fmr(r->r.fmr); 12192e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever if (rc) 12202e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever dprintk("RPC: %s: ib_dealloc_fmr failed %i\n", 12212e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever __func__, rc); 12222e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 12232e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever kfree(r); 12242e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever } 12252e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever} 12262e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 12272e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Leverstatic void 12282e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Leverrpcrdma_destroy_frmrs(struct rpcrdma_buffer *buf) 12292e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever{ 12302e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever struct rpcrdma_mw *r; 12312e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever int rc; 12322e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 12332e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever while (!list_empty(&buf->rb_all)) { 12342e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever r = list_entry(buf->rb_all.next, struct rpcrdma_mw, mw_all); 12352e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever list_del(&r->mw_all); 12362e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever list_del(&r->mw_list); 12372e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 12382e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever rc = ib_dereg_mr(r->r.frmr.fr_mr); 12392e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever if (rc) 12402e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever dprintk("RPC: %s: ib_dereg_mr failed %i\n", 12412e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever __func__, rc); 12422e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever ib_free_fast_reg_page_list(r->r.frmr.fr_pgl); 12432e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 12442e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever kfree(r); 12452e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever } 12462e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever} 12472e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever 1248c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void 1249c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf) 1250c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1251c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = rdmab_to_ia(buf); 12522e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever int i; 1253c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1254c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* clean up in reverse order from create 1255c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 1. recv mr memory (mr free, then kfree) 1256c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 2. send mr memory (mr free, then kfree) 12572e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever * 3. MWs 1258c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1259c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: entering\n", __func__); 1260c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1261c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ for (i = 0; i < buf->rb_max_requests; i++) { 1262c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (buf->rb_recv_bufs && buf->rb_recv_bufs[i]) { 1263c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_deregister_internal(ia, 1264c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_recv_bufs[i]->rr_handle, 1265c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_recv_bufs[i]->rr_iov); 1266c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ kfree(buf->rb_recv_bufs[i]); 1267c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1268c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (buf->rb_send_bufs && buf->rb_send_bufs[i]) { 1269c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_deregister_internal(ia, 1270c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_send_bufs[i]->rl_handle, 1271c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_send_bufs[i]->rl_iov); 1272c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ kfree(buf->rb_send_bufs[i]); 1273c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1274c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1275c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 12762e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever switch (ia->ri_memreg_strategy) { 12772e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever case RPCRDMA_FRMR: 12782e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever rpcrdma_destroy_frmrs(buf); 12792e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever break; 12802e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever case RPCRDMA_MTHCAFMR: 12812e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever rpcrdma_destroy_fmrs(buf); 12822e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever break; 12832e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever default: 12842e84522c2e0323a090fe1f7eeed6d5b6a68efe5fChuck Lever break; 12854034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews } 12864034ba04231f554abb97ad8900a4c1af03f8e21dAllen Andrews 1287c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ kfree(buf->rb_pool); 1288c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1289c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 12909f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever/* After a disconnect, a flushed FAST_REG_MR can leave an FRMR in 12919f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever * an unusable state. Find FRMRs in this state and dereg / reg 12929f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever * each. FRMRs that are VALID and attached to an rpcrdma_req are 12939f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever * also torn down. 12949f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever * 12959f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever * This gives all in-use FRMRs a fresh rkey and leaves them INVALID. 12969f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever * 12979f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever * This is invoked only in the transport connect worker in order 12989f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever * to serialize with rpcrdma_register_frmr_external(). 12999f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever */ 13009f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Leverstatic void 13019f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Leverrpcrdma_reset_frmrs(struct rpcrdma_ia *ia) 13029f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever{ 13039f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever struct rpcrdma_xprt *r_xprt = 13049f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever container_of(ia, struct rpcrdma_xprt, rx_ia); 13059f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever struct rpcrdma_buffer *buf = &r_xprt->rx_buf; 13069f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever struct list_head *pos; 13079f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever struct rpcrdma_mw *r; 13089f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever int rc; 13099f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever 13109f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever list_for_each(pos, &buf->rb_all) { 13119f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever r = list_entry(pos, struct rpcrdma_mw, mw_all); 13129f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever 13139f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever if (r->r.frmr.fr_state == FRMR_IS_INVALID) 13149f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever continue; 13159f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever 13169f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever rc = ib_dereg_mr(r->r.frmr.fr_mr); 13179f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever if (rc) 13189f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever dprintk("RPC: %s: ib_dereg_mr failed %i\n", 13199f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever __func__, rc); 13209f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever ib_free_fast_reg_page_list(r->r.frmr.fr_pgl); 13219f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever 13229f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever r->r.frmr.fr_mr = ib_alloc_fast_reg_mr(ia->ri_pd, 13239f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever ia->ri_max_frmr_depth); 13249f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever if (IS_ERR(r->r.frmr.fr_mr)) { 13259f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever rc = PTR_ERR(r->r.frmr.fr_mr); 13269f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever dprintk("RPC: %s: ib_alloc_fast_reg_mr" 13279f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever " failed %i\n", __func__, rc); 13289f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever continue; 13299f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever } 13309f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever r->r.frmr.fr_pgl = ib_alloc_fast_reg_page_list( 13319f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever ia->ri_id->device, 13329f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever ia->ri_max_frmr_depth); 13339f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever if (IS_ERR(r->r.frmr.fr_pgl)) { 13349f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever rc = PTR_ERR(r->r.frmr.fr_pgl); 13359f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever dprintk("RPC: %s: " 13369f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever "ib_alloc_fast_reg_page_list " 13379f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever "failed %i\n", __func__, rc); 13389f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever 13399f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever ib_dereg_mr(r->r.frmr.fr_mr); 13409f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever continue; 13419f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever } 13429f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever r->r.frmr.fr_state = FRMR_IS_INVALID; 13439f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever } 13449f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever} 13459f9d802a28a107937ecda4ff78de2ab5cedd439dChuck Lever 1346c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever/* "*mw" can be NULL when rpcrdma_buffer_get_mrs() fails, leaving 1347c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever * some req segments uninitialized. 1348c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever */ 1349c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Leverstatic void 1350c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Leverrpcrdma_buffer_put_mr(struct rpcrdma_mw **mw, struct rpcrdma_buffer *buf) 1351c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever{ 1352c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever if (*mw) { 1353c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever list_add_tail(&(*mw)->mw_list, &buf->rb_mws); 1354c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever *mw = NULL; 1355c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever } 1356c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever} 1357c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever 1358c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever/* Cycle mw's back in reverse order, and "spin" them. 1359c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever * This delays and scrambles reuse as much as possible. 1360c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever */ 1361c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Leverstatic void 1362c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Leverrpcrdma_buffer_put_mrs(struct rpcrdma_req *req, struct rpcrdma_buffer *buf) 1363c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever{ 1364c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever struct rpcrdma_mr_seg *seg = req->rl_segments; 1365c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever struct rpcrdma_mr_seg *seg1 = seg; 1366c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever int i; 1367c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever 1368c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever for (i = 1, seg++; i < RPCRDMA_MAX_SEGS; seg++, i++) 1369c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever rpcrdma_buffer_put_mr(&seg->mr_chunk.rl_mw, buf); 1370c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever rpcrdma_buffer_put_mr(&seg1->mr_chunk.rl_mw, buf); 1371c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever} 1372c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever 1373c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Leverstatic void 1374c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Leverrpcrdma_buffer_put_sendbuf(struct rpcrdma_req *req, struct rpcrdma_buffer *buf) 1375c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever{ 1376c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever buf->rb_send_bufs[--buf->rb_send_index] = req; 1377c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever req->rl_niovs = 0; 1378c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever if (req->rl_reply) { 1379c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever buf->rb_recv_bufs[--buf->rb_recv_index] = req->rl_reply; 1380c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever req->rl_reply->rr_func = NULL; 1381c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever req->rl_reply = NULL; 1382c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever } 1383c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever} 1384c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever 1385ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever/* rpcrdma_unmap_one() was already done by rpcrdma_deregister_frmr_external(). 1386ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever * Redo only the ib_post_send(). 1387ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever */ 1388ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Leverstatic void 1389ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Leverrpcrdma_retry_local_inv(struct rpcrdma_mw *r, struct rpcrdma_ia *ia) 1390ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever{ 1391ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever struct rpcrdma_xprt *r_xprt = 1392ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever container_of(ia, struct rpcrdma_xprt, rx_ia); 1393ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever struct ib_send_wr invalidate_wr, *bad_wr; 1394ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever int rc; 1395ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever 1396ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever dprintk("RPC: %s: FRMR %p is stale\n", __func__, r); 1397ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever 1398ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever /* When this FRMR is re-inserted into rb_mws, it is no longer stale */ 1399dab7e3b8da5ef76143a7e609612c306898f8f8fcChuck Lever r->r.frmr.fr_state = FRMR_IS_INVALID; 1400ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever 1401ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever memset(&invalidate_wr, 0, sizeof(invalidate_wr)); 1402ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever invalidate_wr.wr_id = (unsigned long)(void *)r; 1403ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever invalidate_wr.opcode = IB_WR_LOCAL_INV; 1404ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever invalidate_wr.ex.invalidate_rkey = r->r.frmr.fr_mr->rkey; 1405ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever DECR_CQCOUNT(&r_xprt->rx_ep); 1406ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever 1407ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever dprintk("RPC: %s: frmr %p invalidating rkey %08x\n", 1408ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever __func__, r, r->r.frmr.fr_mr->rkey); 1409ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever 1410ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever read_lock(&ia->ri_qplock); 1411ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever rc = ib_post_send(ia->ri_id->qp, &invalidate_wr, &bad_wr); 1412ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever read_unlock(&ia->ri_qplock); 1413ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever if (rc) { 1414ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever /* Force rpcrdma_buffer_get() to retry */ 1415ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever r->r.frmr.fr_state = FRMR_IS_STALE; 1416ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever dprintk("RPC: %s: ib_post_send failed, %i\n", 1417ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever __func__, rc); 1418ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever } 1419ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever} 1420ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever 1421ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Leverstatic void 1422ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Leverrpcrdma_retry_flushed_linv(struct list_head *stale, 1423ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever struct rpcrdma_buffer *buf) 1424ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever{ 1425ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever struct rpcrdma_ia *ia = rdmab_to_ia(buf); 1426ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever struct list_head *pos; 1427ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever struct rpcrdma_mw *r; 1428ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever unsigned long flags; 1429ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever 1430ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever list_for_each(pos, stale) { 1431ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever r = list_entry(pos, struct rpcrdma_mw, mw_list); 1432ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever rpcrdma_retry_local_inv(r, ia); 1433ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever } 1434ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever 1435ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever spin_lock_irqsave(&buf->rb_lock, flags); 1436ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever list_splice_tail(stale, &buf->rb_mws); 1437ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever spin_unlock_irqrestore(&buf->rb_lock, flags); 1438ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever} 1439ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever 1440ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Leverstatic struct rpcrdma_req * 1441ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Leverrpcrdma_buffer_get_frmrs(struct rpcrdma_req *req, struct rpcrdma_buffer *buf, 1442ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever struct list_head *stale) 1443ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever{ 1444ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever struct rpcrdma_mw *r; 1445ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever int i; 1446ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever 1447ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever i = RPCRDMA_MAX_SEGS - 1; 1448ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever while (!list_empty(&buf->rb_mws)) { 1449ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever r = list_entry(buf->rb_mws.next, 1450ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever struct rpcrdma_mw, mw_list); 1451ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever list_del(&r->mw_list); 1452ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever if (r->r.frmr.fr_state == FRMR_IS_STALE) { 1453ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever list_add(&r->mw_list, stale); 1454ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever continue; 1455ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever } 1456ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever req->rl_segments[i].mr_chunk.rl_mw = r; 1457ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever if (unlikely(i-- == 0)) 1458ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever return req; /* Success */ 1459ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever } 1460ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever 1461ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever /* Not enough entries on rb_mws for this req */ 1462ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever rpcrdma_buffer_put_sendbuf(req, buf); 1463ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever rpcrdma_buffer_put_mrs(req, buf); 1464ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever return NULL; 1465ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever} 1466ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever 1467c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Leverstatic struct rpcrdma_req * 1468ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Leverrpcrdma_buffer_get_fmrs(struct rpcrdma_req *req, struct rpcrdma_buffer *buf) 1469c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever{ 1470c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever struct rpcrdma_mw *r; 1471c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever int i; 1472c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever 1473c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever i = RPCRDMA_MAX_SEGS - 1; 1474c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever while (!list_empty(&buf->rb_mws)) { 1475c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever r = list_entry(buf->rb_mws.next, 1476c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever struct rpcrdma_mw, mw_list); 1477c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever list_del(&r->mw_list); 1478c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever req->rl_segments[i].mr_chunk.rl_mw = r; 1479c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever if (unlikely(i-- == 0)) 1480c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever return req; /* Success */ 1481c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever } 1482c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever 1483c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever /* Not enough entries on rb_mws for this req */ 1484c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever rpcrdma_buffer_put_sendbuf(req, buf); 1485c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever rpcrdma_buffer_put_mrs(req, buf); 1486c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever return NULL; 1487c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever} 1488c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever 1489c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1490c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Get a set of request/reply buffers. 1491c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 1492c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Reply buffer (if needed) is attached to send buffer upon return. 1493c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Rule: 1494c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * rb_send_index and rb_recv_index MUST always be pointing to the 1495c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * *next* available buffer (non-NULL). They are incremented after 1496c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * removing buffers, and decremented *before* returning them. 1497c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1498c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\struct rpcrdma_req * 1499c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_buffer_get(struct rpcrdma_buffer *buffers) 1500c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1501c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever struct rpcrdma_ia *ia = rdmab_to_ia(buffers); 1502ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever struct list_head stale; 1503c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_req *req; 1504c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 1505c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1506c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&buffers->rb_lock, flags); 1507c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (buffers->rb_send_index == buffers->rb_max_requests) { 1508c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&buffers->rb_lock, flags); 1509c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: out of request buffers\n", __func__); 1510c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return ((struct rpcrdma_req *)NULL); 1511c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1512c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1513c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req = buffers->rb_send_bufs[buffers->rb_send_index]; 1514c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (buffers->rb_send_index < buffers->rb_recv_index) { 1515c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: %d extra receives outstanding (ok)\n", 1516c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, 1517c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_recv_index - buffers->rb_send_index); 1518c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply = NULL; 1519c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } else { 1520c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply = buffers->rb_recv_bufs[buffers->rb_recv_index]; 1521c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_recv_bufs[buffers->rb_recv_index++] = NULL; 1522c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1523c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_send_bufs[buffers->rb_send_index++] = NULL; 1524ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever 1525ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever INIT_LIST_HEAD(&stale); 1526c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever switch (ia->ri_memreg_strategy) { 1527c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever case RPCRDMA_FRMR: 1528ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever req = rpcrdma_buffer_get_frmrs(req, buffers, &stale); 1529ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever break; 1530c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever case RPCRDMA_MTHCAFMR: 1531ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever req = rpcrdma_buffer_get_fmrs(req, buffers); 1532c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever break; 1533c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever default: 1534c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever break; 1535c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1536c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&buffers->rb_lock, flags); 1537ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever if (!list_empty(&stale)) 1538ddb6bebcc64678fcf73eb9e21f80c6dacfa093a7Chuck Lever rpcrdma_retry_flushed_linv(&stale, buffers); 1539c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return req; 1540c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1541c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1542c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1543c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Put request/reply buffers back into pool. 1544c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Pre-decrement counter/array index. 1545c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1546c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void 1547c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_buffer_put(struct rpcrdma_req *req) 1548c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1549c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_buffer *buffers = req->rl_buffer; 1550c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = rdmab_to_ia(buffers); 1551c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 1552c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1553c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&buffers->rb_lock, flags); 1554c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever rpcrdma_buffer_put_sendbuf(req, buffers); 1555c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 15563197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey case RPCRDMA_FRMR: 1557c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 1558c2922c0235aac1c787fa81e24d7d7e93c2202275Chuck Lever rpcrdma_buffer_put_mrs(req, buffers); 1559c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1560c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 1561c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1562c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1563c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&buffers->rb_lock, flags); 1564c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1565c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1566c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1567c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Recover reply buffers from pool. 1568c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * This happens when recovering from error conditions. 1569c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Post-increment counter/array index. 1570c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1571c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void 1572c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_recv_buffer_get(struct rpcrdma_req *req) 1573c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1574c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_buffer *buffers = req->rl_buffer; 1575c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 1576c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1577c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (req->rl_iov.length == 0) /* special case xprt_rdma_allocate() */ 1578c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers = ((struct rpcrdma_req *) buffers)->rl_buffer; 1579c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&buffers->rb_lock, flags); 1580c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (buffers->rb_recv_index < buffers->rb_max_requests) { 1581c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply = buffers->rb_recv_bufs[buffers->rb_recv_index]; 1582c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_recv_bufs[buffers->rb_recv_index++] = NULL; 1583c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1584c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&buffers->rb_lock, flags); 1585c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1586c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1587c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1588c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Put reply buffers back into pool when not attached to 1589b45ccfd25d506e83d9ecf93d0ac7edf031d35d2fChuck Lever * request. This happens in error conditions. 1590c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1591c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void 1592c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep) 1593c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1594c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_buffer *buffers = rep->rr_buffer; 1595c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 1596c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1597c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_func = NULL; 1598c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&buffers->rb_lock, flags); 1599c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_recv_bufs[--buffers->rb_recv_index] = rep; 1600c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&buffers->rb_lock, flags); 1601c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1602c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1603c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1604c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Wrappers for internal-use kmalloc memory registration, used by buffer code. 1605c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1606c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1607c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1608c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_register_internal(struct rpcrdma_ia *ia, void *va, int len, 1609c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_mr **mrp, struct ib_sge *iov) 1610c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1611c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_phys_buf ipb; 1612c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_mr *mr; 1613c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 1614c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1615c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 1616c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * All memory passed here was kmalloc'ed, therefore phys-contiguous. 1617c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1618c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ iov->addr = ib_dma_map_single(ia->ri_id->device, 1619c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ va, len, DMA_BIDIRECTIONAL); 1620bf858ab0adcb36fc60a1d941749a7dd0a9bf3d8aYan Burman if (ib_dma_mapping_error(ia->ri_id->device, iov->addr)) 1621bf858ab0adcb36fc60a1d941749a7dd0a9bf3d8aYan Burman return -ENOMEM; 1622bf858ab0adcb36fc60a1d941749a7dd0a9bf3d8aYan Burman 1623c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ iov->length = len; 1624c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1625bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey if (ia->ri_have_dma_lkey) { 1626bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey *mrp = NULL; 1627bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey iov->lkey = ia->ri_dma_lkey; 1628bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey return 0; 1629bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey } else if (ia->ri_bind_mem != NULL) { 1630c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ *mrp = NULL; 1631c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ iov->lkey = ia->ri_bind_mem->lkey; 1632c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 1633c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1634c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1635c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ipb.addr = iov->addr; 1636c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ipb.size = iov->length; 1637c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ mr = ib_reg_phys_mr(ia->ri_pd, &ipb, 1, 1638c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ IB_ACCESS_LOCAL_WRITE, &iov->addr); 1639c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1640c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: phys convert: 0x%llx " 1641c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "registered 0x%llx length %d\n", 1642a56daeb7d5c9a05b1cb52ae4bcca05fb6545656eAndrew Morton __func__, (unsigned long long)ipb.addr, 1643a56daeb7d5c9a05b1cb52ae4bcca05fb6545656eAndrew Morton (unsigned long long)iov->addr, len); 1644c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1645c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(mr)) { 1646c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ *mrp = NULL; 1647c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(mr); 1648c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: failed with %i\n", __func__, rc); 1649c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } else { 1650c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ *mrp = mr; 1651c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ iov->lkey = mr->lkey; 1652c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = 0; 1653c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1654c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1655c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 1656c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1657c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1658c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1659c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_deregister_internal(struct rpcrdma_ia *ia, 1660c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_mr *mr, struct ib_sge *iov) 1661c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1662c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 1663c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1664c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_unmap_single(ia->ri_id->device, 1665c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ iov->addr, iov->length, DMA_BIDIRECTIONAL); 1666c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1667c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (NULL == mr) 1668c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 1669c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1670c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_dereg_mr(mr); 1671c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1672c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_dereg_mr failed %i\n", __func__, rc); 1673c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 1674c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1675c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1676c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1677c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Wrappers for chunk registration, shared by read/write chunk code. 1678c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1679c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1680c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 1681c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_map_one(struct rpcrdma_ia *ia, struct rpcrdma_mr_seg *seg, int writing) 1682c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1683c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dir = writing ? DMA_FROM_DEVICE : DMA_TO_DEVICE; 1684c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dmalen = seg->mr_len; 1685c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (seg->mr_page) 1686c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dma = ib_dma_map_page(ia->ri_id->device, 1687c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_page, offset_in_page(seg->mr_offset), 1688c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dmalen, seg->mr_dir); 1689c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ else 1690c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dma = ib_dma_map_single(ia->ri_id->device, 1691c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_offset, 1692c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dmalen, seg->mr_dir); 16935c635e09cec0feeeb310968e51dad01040244851Tom Tucker if (ib_dma_mapping_error(ia->ri_id->device, seg->mr_dma)) { 16945c635e09cec0feeeb310968e51dad01040244851Tom Tucker dprintk("RPC: %s: mr_dma %llx mr_offset %p mr_dma_len %zu\n", 16955c635e09cec0feeeb310968e51dad01040244851Tom Tucker __func__, 1696986d4abbddf9e76184f6cabf66654ea8e61bcde5Randy Dunlap (unsigned long long)seg->mr_dma, 1697986d4abbddf9e76184f6cabf66654ea8e61bcde5Randy Dunlap seg->mr_offset, seg->mr_dmalen); 16985c635e09cec0feeeb310968e51dad01040244851Tom Tucker } 1699c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1700c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1701c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 1702c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_unmap_one(struct rpcrdma_ia *ia, struct rpcrdma_mr_seg *seg) 1703c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1704c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (seg->mr_page) 1705c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_unmap_page(ia->ri_id->device, 1706c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dma, seg->mr_dmalen, seg->mr_dir); 1707c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ else 1708c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_unmap_single(ia->ri_id->device, 1709c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dma, seg->mr_dmalen, seg->mr_dir); 1710c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1711c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 17128d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeystatic int 17133197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpeyrpcrdma_register_frmr_external(struct rpcrdma_mr_seg *seg, 17143197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey int *nsegs, int writing, struct rpcrdma_ia *ia, 17153197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey struct rpcrdma_xprt *r_xprt) 17163197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey{ 17173197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey struct rpcrdma_mr_seg *seg1 = seg; 17180dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever struct rpcrdma_mw *mw = seg1->mr_chunk.rl_mw; 17190dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever struct rpcrdma_frmr *frmr = &mw->r.frmr; 17200dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever struct ib_mr *mr = frmr->fr_mr; 1721f590e878c52c38046fd7cfa5a742ddae68717484Chuck Lever struct ib_send_wr fastreg_wr, *bad_wr; 17223197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey u8 key; 17233197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey int len, pageoff; 17243197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey int i, rc; 17259b78145c0f280d4f01c460d6251eab2584181fa9Tom Tucker int seg_len; 17269b78145c0f280d4f01c460d6251eab2584181fa9Tom Tucker u64 pa; 17279b78145c0f280d4f01c460d6251eab2584181fa9Tom Tucker int page_no; 17283197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 17293197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey pageoff = offset_in_page(seg1->mr_offset); 17303197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey seg1->mr_offset -= pageoff; /* start of page */ 17313197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey seg1->mr_len += pageoff; 17323197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey len = -pageoff; 17330fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise if (*nsegs > ia->ri_max_frmr_depth) 17340fc6c4e7bb287148eb5e949efd89327929d4841dSteve Wise *nsegs = ia->ri_max_frmr_depth; 17359b78145c0f280d4f01c460d6251eab2584181fa9Tom Tucker for (page_no = i = 0; i < *nsegs;) { 17363197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey rpcrdma_map_one(ia, seg, writing); 17379b78145c0f280d4f01c460d6251eab2584181fa9Tom Tucker pa = seg->mr_dma; 17389b78145c0f280d4f01c460d6251eab2584181fa9Tom Tucker for (seg_len = seg->mr_len; seg_len > 0; seg_len -= PAGE_SIZE) { 17390dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever frmr->fr_pgl->page_list[page_no++] = pa; 17409b78145c0f280d4f01c460d6251eab2584181fa9Tom Tucker pa += PAGE_SIZE; 17419b78145c0f280d4f01c460d6251eab2584181fa9Tom Tucker } 17423197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey len += seg->mr_len; 17433197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey ++seg; 17443197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey ++i; 17453197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey /* Check for holes */ 17463197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey if ((i < *nsegs && offset_in_page(seg->mr_offset)) || 17473197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len)) 17483197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey break; 17493197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey } 17503197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey dprintk("RPC: %s: Using frmr %p to map %d segments\n", 17510dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever __func__, mw, i); 17523197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 1753050557220e34ed5acc830c9bf6cd993f6b4ea33eChuck Lever frmr->fr_state = FRMR_IS_VALID; 1754050557220e34ed5acc830c9bf6cd993f6b4ea33eChuck Lever 1755f590e878c52c38046fd7cfa5a742ddae68717484Chuck Lever memset(&fastreg_wr, 0, sizeof(fastreg_wr)); 1756f590e878c52c38046fd7cfa5a742ddae68717484Chuck Lever fastreg_wr.wr_id = (unsigned long)(void *)mw; 1757f590e878c52c38046fd7cfa5a742ddae68717484Chuck Lever fastreg_wr.opcode = IB_WR_FAST_REG_MR; 1758f590e878c52c38046fd7cfa5a742ddae68717484Chuck Lever fastreg_wr.wr.fast_reg.iova_start = seg1->mr_dma; 1759f590e878c52c38046fd7cfa5a742ddae68717484Chuck Lever fastreg_wr.wr.fast_reg.page_list = frmr->fr_pgl; 1760f590e878c52c38046fd7cfa5a742ddae68717484Chuck Lever fastreg_wr.wr.fast_reg.page_list_len = page_no; 1761f590e878c52c38046fd7cfa5a742ddae68717484Chuck Lever fastreg_wr.wr.fast_reg.page_shift = PAGE_SHIFT; 1762f590e878c52c38046fd7cfa5a742ddae68717484Chuck Lever fastreg_wr.wr.fast_reg.length = page_no << PAGE_SHIFT; 1763f590e878c52c38046fd7cfa5a742ddae68717484Chuck Lever if (fastreg_wr.wr.fast_reg.length < len) { 17645fc83f470d8ada25927701512cf94a53dab6c4c8Chuck Lever rc = -EIO; 17655fc83f470d8ada25927701512cf94a53dab6c4c8Chuck Lever goto out_err; 1766c977dea22708688eae31774f70126c97aa4dfe83Chuck Lever } 1767c977dea22708688eae31774f70126c97aa4dfe83Chuck Lever 1768c977dea22708688eae31774f70126c97aa4dfe83Chuck Lever /* Bump the key */ 17690dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever key = (u8)(mr->rkey & 0x000000FF); 17700dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever ib_update_fast_reg_key(mr, ++key); 1771c977dea22708688eae31774f70126c97aa4dfe83Chuck Lever 1772f590e878c52c38046fd7cfa5a742ddae68717484Chuck Lever fastreg_wr.wr.fast_reg.access_flags = (writing ? 177368743082b560067e3e93eab8b2568f238e486865Vu Pham IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE : 177468743082b560067e3e93eab8b2568f238e486865Vu Pham IB_ACCESS_REMOTE_READ); 1775f590e878c52c38046fd7cfa5a742ddae68717484Chuck Lever fastreg_wr.wr.fast_reg.rkey = mr->rkey; 17763197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey DECR_CQCOUNT(&r_xprt->rx_ep); 17773197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 1778f590e878c52c38046fd7cfa5a742ddae68717484Chuck Lever rc = ib_post_send(ia->ri_id->qp, &fastreg_wr, &bad_wr); 17793197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey if (rc) { 17803197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey dprintk("RPC: %s: failed ib_post_send for register," 17813197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey " status %i\n", __func__, rc); 1782c93e986a295d537589efd0504f36ca952bd1a5beChuck Lever ib_update_fast_reg_key(mr, --key); 17835fc83f470d8ada25927701512cf94a53dab6c4c8Chuck Lever goto out_err; 17843197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey } else { 17850dbb4108a6a615589751de2aaf468d3ddbcef24cChuck Lever seg1->mr_rkey = mr->rkey; 17863197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey seg1->mr_base = seg1->mr_dma + pageoff; 17873197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey seg1->mr_nsegs = i; 17883197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey seg1->mr_len = len; 17893197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey } 17903197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey *nsegs = i; 17915fc83f470d8ada25927701512cf94a53dab6c4c8Chuck Lever return 0; 17925fc83f470d8ada25927701512cf94a53dab6c4c8Chuck Leverout_err: 1793050557220e34ed5acc830c9bf6cd993f6b4ea33eChuck Lever frmr->fr_state = FRMR_IS_INVALID; 17945fc83f470d8ada25927701512cf94a53dab6c4c8Chuck Lever while (i--) 17955fc83f470d8ada25927701512cf94a53dab6c4c8Chuck Lever rpcrdma_unmap_one(ia, --seg); 17963197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey return rc; 17973197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey} 17983197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 17993197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpeystatic int 18003197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpeyrpcrdma_deregister_frmr_external(struct rpcrdma_mr_seg *seg, 18013197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey struct rpcrdma_ia *ia, struct rpcrdma_xprt *r_xprt) 18023197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey{ 18033197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey struct rpcrdma_mr_seg *seg1 = seg; 18043197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey struct ib_send_wr invalidate_wr, *bad_wr; 18053197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey int rc; 18063197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 1807dab7e3b8da5ef76143a7e609612c306898f8f8fcChuck Lever seg1->mr_chunk.rl_mw->r.frmr.fr_state = FRMR_IS_INVALID; 1808dab7e3b8da5ef76143a7e609612c306898f8f8fcChuck Lever 18093197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey memset(&invalidate_wr, 0, sizeof invalidate_wr); 18105c635e09cec0feeeb310968e51dad01040244851Tom Tucker invalidate_wr.wr_id = (unsigned long)(void *)seg1->mr_chunk.rl_mw; 18113197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey invalidate_wr.opcode = IB_WR_LOCAL_INV; 18123197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey invalidate_wr.ex.invalidate_rkey = seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey; 18133197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey DECR_CQCOUNT(&r_xprt->rx_ep); 18143197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 181573806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever read_lock(&ia->ri_qplock); 181673806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever while (seg1->mr_nsegs--) 181773806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever rpcrdma_unmap_one(ia, seg++); 18183197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey rc = ib_post_send(ia->ri_id->qp, &invalidate_wr, &bad_wr); 181973806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever read_unlock(&ia->ri_qplock); 1820dab7e3b8da5ef76143a7e609612c306898f8f8fcChuck Lever if (rc) { 1821dab7e3b8da5ef76143a7e609612c306898f8f8fcChuck Lever /* Force rpcrdma_buffer_get() to retry */ 1822dab7e3b8da5ef76143a7e609612c306898f8f8fcChuck Lever seg1->mr_chunk.rl_mw->r.frmr.fr_state = FRMR_IS_STALE; 18233197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey dprintk("RPC: %s: failed ib_post_send for invalidate," 18243197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey " status %i\n", __func__, rc); 1825dab7e3b8da5ef76143a7e609612c306898f8f8fcChuck Lever } 18263197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey return rc; 18273197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey} 18283197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 18293197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpeystatic int 18308d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeyrpcrdma_register_fmr_external(struct rpcrdma_mr_seg *seg, 18318d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int *nsegs, int writing, struct rpcrdma_ia *ia) 18328d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey{ 18338d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_mr_seg *seg1 = seg; 18348d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey u64 physaddrs[RPCRDMA_MAX_DATA_SEGS]; 18358d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int len, pageoff, i, rc; 18368d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 18378d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey pageoff = offset_in_page(seg1->mr_offset); 18388d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_offset -= pageoff; /* start of page */ 18398d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_len += pageoff; 18408d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey len = -pageoff; 18418d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (*nsegs > RPCRDMA_MAX_DATA_SEGS) 18428d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey *nsegs = RPCRDMA_MAX_DATA_SEGS; 18438d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey for (i = 0; i < *nsegs;) { 18448d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_map_one(ia, seg, writing); 18458d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey physaddrs[i] = seg->mr_dma; 18468d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey len += seg->mr_len; 18478d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey ++seg; 18488d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey ++i; 18498d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey /* Check for holes */ 18508d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if ((i < *nsegs && offset_in_page(seg->mr_offset)) || 18518d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len)) 18528d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey break; 18538d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } 18548d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = ib_map_phys_fmr(seg1->mr_chunk.rl_mw->r.fmr, 18558d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey physaddrs, i, seg1->mr_dma); 18568d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (rc) { 18578d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey dprintk("RPC: %s: failed ib_map_phys_fmr " 18588d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey "%u@0x%llx+%i (%d)... status %i\n", __func__, 18598d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey len, (unsigned long long)seg1->mr_dma, 18608d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey pageoff, i, rc); 18618d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey while (i--) 18628d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_unmap_one(ia, --seg); 18638d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } else { 18648d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_rkey = seg1->mr_chunk.rl_mw->r.fmr->rkey; 18658d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_base = seg1->mr_dma + pageoff; 18668d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_nsegs = i; 18678d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_len = len; 18688d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } 18698d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey *nsegs = i; 18708d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey return rc; 18718d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey} 18728d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 18738d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeystatic int 18748d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeyrpcrdma_deregister_fmr_external(struct rpcrdma_mr_seg *seg, 18758d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_ia *ia) 18768d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey{ 18778d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_mr_seg *seg1 = seg; 18788d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey LIST_HEAD(l); 18798d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int rc; 18808d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 18818d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey list_add(&seg1->mr_chunk.rl_mw->r.fmr->list, &l); 18828d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = ib_unmap_fmr(&l); 188373806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever read_lock(&ia->ri_qplock); 18848d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey while (seg1->mr_nsegs--) 18858d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_unmap_one(ia, seg++); 188673806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever read_unlock(&ia->ri_qplock); 18878d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (rc) 18888d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey dprintk("RPC: %s: failed ib_unmap_fmr," 18898d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey " status %i\n", __func__, rc); 18908d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey return rc; 18918d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey} 18928d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 1893c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1894c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_register_external(struct rpcrdma_mr_seg *seg, 1895c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int nsegs, int writing, struct rpcrdma_xprt *r_xprt) 1896c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1897c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = &r_xprt->rx_ia; 1898c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc = 0; 1899c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1900c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 1901c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1902c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_ALLPHYSICAL: 1903c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_map_one(ia, seg, writing); 1904c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_rkey = ia->ri_bind_mem->rkey; 1905c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_base = seg->mr_dma; 1906c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_nsegs = 1; 1907c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ nsegs = 1; 1908c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1909c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 19103197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey /* Registration using frmr registration */ 19113197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey case RPCRDMA_FRMR: 19123197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey rc = rpcrdma_register_frmr_external(seg, &nsegs, writing, ia, r_xprt); 19133197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey break; 19143197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 19158d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey /* Registration using fmr memory registration */ 1916c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 19178d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = rpcrdma_register_fmr_external(seg, &nsegs, writing, ia); 1918c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1919c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1920c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 19210ac531c1832318efa3dc3d723e356a7e09330e80Chuck Lever return -1; 1922c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1923c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1924c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return -1; 1925c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1926c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return nsegs; 1927c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1928c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1929c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1930c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_deregister_external(struct rpcrdma_mr_seg *seg, 193113c9ff8f673862b69e795ea99a237b461c557eb3Chuck Lever struct rpcrdma_xprt *r_xprt) 1932c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1933c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = &r_xprt->rx_ia; 1934c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int nsegs = seg->mr_nsegs, rc; 1935c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1936c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 1937c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1938c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_ALLPHYSICAL: 193973806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever read_lock(&ia->ri_qplock); 1940c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_unmap_one(ia, seg); 194173806c8832b3438ef0439603dab1f3cfc61cb6cdChuck Lever read_unlock(&ia->ri_qplock); 1942c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1943c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 19443197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey case RPCRDMA_FRMR: 19453197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey rc = rpcrdma_deregister_frmr_external(seg, ia, r_xprt); 19463197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey break; 19473197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 1948c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 19498d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = rpcrdma_deregister_fmr_external(seg, ia); 1950c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1951c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1952c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 1953c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1954c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1955c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return nsegs; 1956c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1957c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1958c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1959c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Prepost any receive buffer, then post send. 1960c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 1961c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Receive buffer is donated to hardware, reclaimed upon recv completion. 1962c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1963c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1964c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_post(struct rpcrdma_ia *ia, 1965c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ep *ep, 1966c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_req *req) 1967c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1968c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_send_wr send_wr, *send_wr_fail; 1969c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep *rep = req->rl_reply; 1970c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 1971c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1972c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rep) { 1973c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_ep_post_recv(ia, ep, rep); 1974c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1975c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1976c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply = NULL; 1977c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1978c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1979c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.next = NULL; 1980c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.wr_id = 0ULL; /* no send cookie */ 1981c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.sg_list = req->rl_send_iov; 1982c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.num_sge = req->rl_niovs; 1983c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.opcode = IB_WR_SEND; 1984c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (send_wr.num_sge == 4) /* no need to sync any pad (constant) */ 1985c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_sync_single_for_device(ia->ri_id->device, 1986c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_send_iov[3].addr, req->rl_send_iov[3].length, 1987c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ DMA_TO_DEVICE); 1988c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_sync_single_for_device(ia->ri_id->device, 1989c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_send_iov[1].addr, req->rl_send_iov[1].length, 1990c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ DMA_TO_DEVICE); 1991c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_sync_single_for_device(ia->ri_id->device, 1992c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_send_iov[0].addr, req->rl_send_iov[0].length, 1993c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ DMA_TO_DEVICE); 1994c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1995c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (DECR_CQCOUNT(ep) > 0) 1996c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.send_flags = 0; 1997c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ else { /* Provider must take a send completion every now and then */ 1998c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ INIT_CQCOUNT(ep); 1999c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.send_flags = IB_SEND_SIGNALED; 2000c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 2001c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 2002c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_post_send(ia->ri_id->qp, &send_wr, &send_wr_fail); 2003c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 2004c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_post_send returned %i\n", __func__, 2005c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc); 2006c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out: 2007c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 2008c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 2009c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 2010c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 2011c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * (Re)post a receive buffer. 2012c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 2013c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 2014c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_post_recv(struct rpcrdma_ia *ia, 2015c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ep *ep, 2016c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep *rep) 2017c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 2018c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_recv_wr recv_wr, *recv_wr_fail; 2019c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 2020c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 2021c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ recv_wr.next = NULL; 2022c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ recv_wr.wr_id = (u64) (unsigned long) rep; 2023c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ recv_wr.sg_list = &rep->rr_iov; 2024c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ recv_wr.num_sge = 1; 2025c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 2026c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_sync_single_for_cpu(ia->ri_id->device, 2027c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_iov.addr, rep->rr_iov.length, DMA_BIDIRECTIONAL); 2028c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 2029c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_post_recv(ia->ri_id->qp, &recv_wr, &recv_wr_fail); 2030c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 2031c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 2032c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_post_recv returned %i\n", __func__, 2033c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc); 2034c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 2035c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 203643e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever 203743e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever/* Physical mapping means one Read/Write list entry per-page. 203843e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever * All list entries must fit within an inline buffer 203943e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever * 204043e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever * NB: The server must return a Write list for NFS READ, 204143e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever * which has the same constraint. Factor in the inline 204243e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever * rsize as well. 204343e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever */ 204443e95988178ed70a878a5be6be9ad248342dbf7dChuck Leverstatic size_t 204543e95988178ed70a878a5be6be9ad248342dbf7dChuck Leverrpcrdma_physical_max_payload(struct rpcrdma_xprt *r_xprt) 204643e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever{ 204743e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever struct rpcrdma_create_data_internal *cdata = &r_xprt->rx_data; 204843e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever unsigned int inline_size, pages; 204943e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever 205043e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever inline_size = min_t(unsigned int, 205143e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever cdata->inline_wsize, cdata->inline_rsize); 205243e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever inline_size -= RPCRDMA_HDRLEN_MIN; 205343e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever pages = inline_size / sizeof(struct rpcrdma_segment); 205443e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever return pages << PAGE_SHIFT; 205543e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever} 205643e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever 205743e95988178ed70a878a5be6be9ad248342dbf7dChuck Leverstatic size_t 205843e95988178ed70a878a5be6be9ad248342dbf7dChuck Leverrpcrdma_mr_max_payload(struct rpcrdma_xprt *r_xprt) 205943e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever{ 206043e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever return RPCRDMA_MAX_DATA_SEGS << PAGE_SHIFT; 206143e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever} 206243e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever 206343e95988178ed70a878a5be6be9ad248342dbf7dChuck Leversize_t 206443e95988178ed70a878a5be6be9ad248342dbf7dChuck Leverrpcrdma_max_payload(struct rpcrdma_xprt *r_xprt) 206543e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever{ 206643e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever size_t result; 206743e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever 206843e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever switch (r_xprt->rx_ia.ri_memreg_strategy) { 206943e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever case RPCRDMA_ALLPHYSICAL: 207043e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever result = rpcrdma_physical_max_payload(r_xprt); 207143e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever break; 207243e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever default: 207343e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever result = rpcrdma_mr_max_payload(r_xprt); 207443e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever } 207543e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever return result; 207643e95988178ed70a878a5be6be9ad248342dbf7dChuck Lever} 2077