verbs.c revision b3cd8d45a764e6edb06e7bd386faf99a879569b8
1f58851e6b0f148fb4b2a1c6f70beb2f125863c0f\"Talpey, Thomas\/* 2c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved. 3c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 4c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * This software is available to you under a choice of one of two 5c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * licenses. You may choose to be licensed under the terms of the GNU 6c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * General Public License (GPL) Version 2, available from the file 7c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * COPYING in the main directory of this source tree, or the BSD-type 8c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * license below: 9c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 10c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Redistribution and use in source and binary forms, with or without 11c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * modification, are permitted provided that the following conditions 12c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * are met: 13c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 14c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Redistributions of source code must retain the above copyright 15c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * notice, this list of conditions and the following disclaimer. 16c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 17c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Redistributions in binary form must reproduce the above 18c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * copyright notice, this list of conditions and the following 19c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * disclaimer in the documentation and/or other materials provided 20c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * with the distribution. 21c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 22c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Neither the name of the Network Appliance, Inc. nor the names of 23c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * its contributors may be used to endorse or promote products 24c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * derived from this software without specific prior written 25c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * permission. 26c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 27c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 28c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 29c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 30c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 31c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 32c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 33c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 34c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 35c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 36c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 37c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 38f58851e6b0f148fb4b2a1c6f70beb2f125863c0f\"Talpey, Thomas\ */ 39f58851e6b0f148fb4b2a1c6f70beb2f125863c0f\"Talpey, Thomas\ 40c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 41c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * verbs.c 42c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 43c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Encapsulates the major functions managing: 44c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o adapters 45c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o endpoints 46c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o connections 47c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o buffer memory 48c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 49c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 50c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#include <linux/pci.h> /* for Tavor hack below */ 51c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 52f58851e6b0f148fb4b2a1c6f70beb2f125863c0f\"Talpey, Thomas\#include "xprt_rdma.h" 53f58851e6b0f148fb4b2a1c6f70beb2f125863c0f\"Talpey, Thomas\ 54c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 55c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Globals/Macros 56c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 57c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 58c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#ifdef RPC_DEBUG 59c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\# define RPCDBG_FACILITY RPCDBG_TRANS 60c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#endif 61c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 62c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 63c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * internal functions 64c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 65c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 66c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 67c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * handle replies in tasklet context, using a single, global list 68c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * rdma tasklet function -- just turn around and call the func 69c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * for all replies on the list 70c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 71c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 72c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static DEFINE_SPINLOCK(rpcrdma_tk_lock_g); 73c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static LIST_HEAD(rpcrdma_tasklets_g); 74c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 75c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 76c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_run_tasklet(unsigned long data) 77c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 78c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep *rep; 79c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ void (*func)(struct rpcrdma_rep *); 80c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 81c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 82c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ data = data; 83c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&rpcrdma_tk_lock_g, flags); 84c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ while (!list_empty(&rpcrdma_tasklets_g)) { 85c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep = list_entry(rpcrdma_tasklets_g.next, 86c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep, rr_list); 87c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_del(&rep->rr_list); 88c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ func = rep->rr_func; 89c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_func = NULL; 90c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&rpcrdma_tk_lock_g, flags); 91c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 92c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (func) 93c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ func(rep); 94c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ else 95c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_recv_buffer_put(rep); 96c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 97c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&rpcrdma_tk_lock_g, flags); 98c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 99c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&rpcrdma_tk_lock_g, flags); 100c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 101c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 102c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static DECLARE_TASKLET(rpcrdma_tasklet_g, rpcrdma_run_tasklet, 0UL); 103c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 104c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static inline void 105c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_schedule_tasklet(struct rpcrdma_rep *rep) 106c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 107c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 108c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 109c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&rpcrdma_tk_lock_g, flags); 110c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_add_tail(&rep->rr_list, &rpcrdma_tasklets_g); 111c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&rpcrdma_tk_lock_g, flags); 112c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ tasklet_schedule(&rpcrdma_tasklet_g); 113c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 114c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 115c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 116c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_qp_async_error_upcall(struct ib_event *event, void *context) 117c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 118c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ep *ep = context; 119c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 120c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: QP error %X on device %s ep %p\n", 121c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, event->event, event->device->name, context); 122c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_connected == 1) { 123c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = -EIO; 124c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_func(ep); 125c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wake_up_all(&ep->rep_connect_wait); 126c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 127c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 128c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 129c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 130c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_cq_async_error_upcall(struct ib_event *event, void *context) 131c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 132c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ep *ep = context; 133c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 134c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: CQ error %X on device %s ep %p\n", 135c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, event->event, event->device->name, context); 136c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_connected == 1) { 137c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = -EIO; 138c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_func(ep); 139c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wake_up_all(&ep->rep_connect_wait); 140c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 141c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 142c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 143c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static inline 144c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void rpcrdma_event_process(struct ib_wc *wc) 145c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 146c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep *rep = 147c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (struct rpcrdma_rep *)(unsigned long) wc->wr_id; 148c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 149c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: event rep %p status %X opcode %X length %u\n", 150c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rep, wc->status, wc->opcode, wc->byte_len); 151c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 152c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (!rep) /* send or bind completion that we don't care about */ 153c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return; 154c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 155c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IB_WC_SUCCESS != wc->status) { 156c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: %s WC status %X, connection lost\n", 157c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, (wc->opcode & IB_WC_RECV) ? "recv" : "send", 158c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wc->status); 159c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_len = ~0U; 160c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_schedule_tasklet(rep); 161c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return; 162c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 163c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 164c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (wc->opcode) { 165c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case IB_WC_RECV: 166c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_len = wc->byte_len; 167c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_sync_single_for_cpu( 168c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rdmab_to_ia(rep->rr_buffer)->ri_id->device, 169c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_iov.addr, rep->rr_len, DMA_FROM_DEVICE); 170c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Keep (only) the most recent credits, after check validity */ 171c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rep->rr_len >= 16) { 172c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_msg *p = 173c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (struct rpcrdma_msg *) rep->rr_base; 174c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned int credits = ntohl(p->rm_credit); 175c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (credits == 0) { 176c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: server" 177c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " dropped credits to 0!\n", __func__); 178c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* don't deadlock */ 179c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ credits = 1; 180c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } else if (credits > rep->rr_buffer->rb_max_requests) { 181c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: server" 182c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " over-crediting: %d (%d)\n", 183c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, credits, 184c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_buffer->rb_max_requests); 185c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ credits = rep->rr_buffer->rb_max_requests; 186c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 187c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ atomic_set(&rep->rr_buffer->rb_credits, credits); 188c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 189c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* fall through */ 190c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case IB_WC_BIND_MW: 191c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_schedule_tasklet(rep); 192c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 193c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 194c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: unexpected WC event %X\n", 195c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, wc->opcode); 196c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 197c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 198c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 199c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 200c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static inline int 201c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_cq_poll(struct ib_cq *cq) 202c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 203c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_wc wc; 204c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 205c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 206c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ for (;;) { 207c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_poll_cq(cq, 1, &wc); 208c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc < 0) { 209c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_poll_cq failed %i\n", 210c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 211c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 212c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 213c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc == 0) 214c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 215c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 216c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_event_process(&wc); 217c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 218c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 219c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 220c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 221c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 222c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 223c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * rpcrdma_cq_event_upcall 224c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 225c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * This upcall handles recv, send, bind and unbind events. 226c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * It is reentrant but processes single events in order to maintain 227c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * ordering of receives to keep server credits. 228c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 229c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * It is the responsibility of the scheduled tasklet to return 230c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * recv buffers to the pool. NOTE: this affects synchronization of 231c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * connection shutdown. That is, the structures required for 232c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * the completion of the reply handler must remain intact until 233c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * all memory has been reclaimed. 234c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 235c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Note that send events are suppressed and do not result in an upcall. 236c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 237c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 238c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_cq_event_upcall(struct ib_cq *cq, void *context) 239c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 240c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 241c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 242c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_cq_poll(cq); 243c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 244c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return; 245c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 246c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_req_notify_cq(cq, IB_CQ_NEXT_COMP); 247c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 248c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_req_notify_cq failed %i\n", 249c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 250c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return; 251c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 252c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 253c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_cq_poll(cq); 254c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 255c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 256c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#ifdef RPC_DEBUG 257c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static const char * const conn[] = { 258c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "address resolved", 259c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "address error", 260c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "route resolved", 261c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "route error", 262c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "connect request", 263c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "connect response", 264c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "connect error", 265c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "unreachable", 266c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "rejected", 267c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "established", 268c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "disconnected", 269c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "device removal" 270c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\}; 271c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#endif 272c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 273c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static int 274c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_conn_upcall(struct rdma_cm_id *id, struct rdma_cm_event *event) 275c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 276c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_xprt *xprt = id->context; 277c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = &xprt->rx_ia; 278c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ep *ep = &xprt->rx_ep; 279c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct sockaddr_in *addr = (struct sockaddr_in *) &ep->rep_remote_addr; 280c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_qp_attr attr; 281c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_qp_init_attr iattr; 282c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int connstate = 0; 283c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 284c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (event->event) { 285c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_ADDR_RESOLVED: 286c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_ROUTE_RESOLVED: 2875675add36e76b9487e7f9e689f854cb8d6afd9b4Tom Talpey ia->ri_async_rc = 0; 288c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ complete(&ia->ri_done); 289c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 290c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_ADDR_ERROR: 291c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_async_rc = -EHOSTUNREACH; 292c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: CM address resolution error, ep 0x%p\n", 293c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, ep); 294c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ complete(&ia->ri_done); 295c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 296c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_ROUTE_ERROR: 297c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_async_rc = -ENETUNREACH; 298c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: CM route resolution error, ep 0x%p\n", 299c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, ep); 300c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ complete(&ia->ri_done); 301c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 302c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_ESTABLISHED: 303c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = 1; 304c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_query_qp(ia->ri_id->qp, &attr, 305c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ IB_QP_MAX_QP_RD_ATOMIC | IB_QP_MAX_DEST_RD_ATOMIC, 306c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &iattr); 307c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: %d responder resources" 308c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " (%d initiator)\n", 309c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, attr.max_dest_rd_atomic, attr.max_rd_atomic); 310c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto connected; 311c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_CONNECT_ERROR: 312c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = -ENOTCONN; 313c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto connected; 314c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_UNREACHABLE: 315c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = -ENETDOWN; 316c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto connected; 317c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_REJECTED: 318c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = -ECONNREFUSED; 319c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto connected; 320c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_DISCONNECTED: 321c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = -ECONNABORTED; 322c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto connected; 323c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_DEVICE_REMOVAL: 324c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = -ENODEV; 325c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\connected: 326c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: %s: %u.%u.%u.%u:%u" 327c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " (ep 0x%p event 0x%x)\n", 328c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, 329c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (event->event <= 11) ? conn[event->event] : 330c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "unknown connection error", 331c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ NIPQUAD(addr->sin_addr.s_addr), 332c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ntohs(addr->sin_port), 333c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep, event->event); 334c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ atomic_set(&rpcx_to_rdmax(ep->rep_xprt)->rx_buf.rb_credits, 1); 335c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: %sconnected\n", 336c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, connstate > 0 ? "" : "dis"); 337c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = connstate; 338c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_func(ep); 339c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wake_up_all(&ep->rep_connect_wait); 340c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 341c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 3421a954051b0cf79bd67e5f9db40333e3a9b1d05d2Tom Talpey dprintk("RPC: %s: unexpected CM event %d\n", 343c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, event->event); 344c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 345c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 346c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 347b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey#ifdef RPC_DEBUG 348b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey if (connstate == 1) { 349b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey int ird = attr.max_dest_rd_atomic; 350b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey int tird = ep->rep_remote_cma.responder_resources; 351b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey printk(KERN_INFO "rpcrdma: connection to %u.%u.%u.%u:%u " 352b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey "on %s, memreg %d slots %d ird %d%s\n", 353b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey NIPQUAD(addr->sin_addr.s_addr), 354b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey ntohs(addr->sin_port), 355b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey ia->ri_id->device->name, 356b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey ia->ri_memreg_strategy, 357b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey xprt->rx_buf.rb_max_requests, 358b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey ird, ird < 4 && ird < tird / 2 ? " (low!)" : ""); 359b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey } else if (connstate < 0) { 360b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey printk(KERN_INFO "rpcrdma: connection to %u.%u.%u.%u:%u " 361b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey "closed (%d)\n", 362b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey NIPQUAD(addr->sin_addr.s_addr), 363b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey ntohs(addr->sin_port), 364b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey connstate); 365b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey } 366b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey#endif 367b3cd8d45a764e6edb06e7bd386faf99a879569b8Tom Talpey 368c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 369c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 370c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 371c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static struct rdma_cm_id * 372c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_create_id(struct rpcrdma_xprt *xprt, 373c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia, struct sockaddr *addr) 374c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 375c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rdma_cm_id *id; 376c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 377c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 3781a954051b0cf79bd67e5f9db40333e3a9b1d05d2Tom Talpey init_completion(&ia->ri_done); 3791a954051b0cf79bd67e5f9db40333e3a9b1d05d2Tom Talpey 380c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ id = rdma_create_id(rpcrdma_conn_upcall, xprt, RDMA_PS_TCP); 381c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(id)) { 382c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(id); 383c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_create_id() failed %i\n", 384c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 385c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return id; 386c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 387c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 3885675add36e76b9487e7f9e689f854cb8d6afd9b4Tom Talpey ia->ri_async_rc = -ETIMEDOUT; 389c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rdma_resolve_addr(id, NULL, addr, RDMA_RESOLVE_TIMEOUT); 390c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 391c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_resolve_addr() failed %i\n", 392c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 393c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 394c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 3955675add36e76b9487e7f9e689f854cb8d6afd9b4Tom Talpey wait_for_completion_interruptible_timeout(&ia->ri_done, 3965675add36e76b9487e7f9e689f854cb8d6afd9b4Tom Talpey msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1); 397c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ia->ri_async_rc; 398c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 399c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 400c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 4015675add36e76b9487e7f9e689f854cb8d6afd9b4Tom Talpey ia->ri_async_rc = -ETIMEDOUT; 402c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rdma_resolve_route(id, RDMA_RESOLVE_TIMEOUT); 403c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 404c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_resolve_route() failed %i\n", 405c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 406c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 407c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 4085675add36e76b9487e7f9e689f854cb8d6afd9b4Tom Talpey wait_for_completion_interruptible_timeout(&ia->ri_done, 4095675add36e76b9487e7f9e689f854cb8d6afd9b4Tom Talpey msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1); 410c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ia->ri_async_rc; 411c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 412c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 413c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 414c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return id; 415c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 416c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out: 417c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rdma_destroy_id(id); 418c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return ERR_PTR(rc); 419c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 420c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 421c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 422c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Drain any cq, prior to teardown. 423c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 424c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 425c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_clean_cq(struct ib_cq *cq) 426c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 427c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_wc wc; 428c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int count = 0; 429c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 430c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ while (1 == ib_poll_cq(cq, 1, &wc)) 431c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ++count; 432c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 433c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (count) 434c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: flushed %d events (last 0x%x)\n", 435c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, count, wc.opcode); 436c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 437c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 438c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 439c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Exported functions. 440c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 441c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 442c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 443c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Open and initialize an Interface Adapter. 444c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o initializes fields of struct rpcrdma_ia, including 445c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * interface and provider attributes and protection zone. 446c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 447c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 448c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ia_open(struct rpcrdma_xprt *xprt, struct sockaddr *addr, int memreg) 449c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 450bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey int rc, mem_priv; 451bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey struct ib_device_attr devattr; 452c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = &xprt->rx_ia; 453c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 454c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_id = rpcrdma_create_id(xprt, ia, addr); 455c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(ia->ri_id)) { 456c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(ia->ri_id); 457c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out1; 458c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 459c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 460c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_pd = ib_alloc_pd(ia->ri_id->device); 461c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(ia->ri_pd)) { 462c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(ia->ri_pd); 463c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_alloc_pd() failed %i\n", 464c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 465c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out2; 466c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 467c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 468c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 469bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey * Query the device to determine if the requested memory 470bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey * registration strategy is supported. If it isn't, set the 471bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey * strategy to a globally supported model. 472bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey */ 473bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey rc = ib_query_device(ia->ri_id->device, &devattr); 474bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey if (rc) { 475bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey dprintk("RPC: %s: ib_query_device failed %d\n", 476bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey __func__, rc); 477bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey goto out2; 478bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey } 479bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey 480bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey if (devattr.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY) { 481bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey ia->ri_have_dma_lkey = 1; 482bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey ia->ri_dma_lkey = ia->ri_id->device->local_dma_lkey; 483bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey } 484bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey 485bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey switch (memreg) { 486bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_MEMWINDOWS: 487bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_MEMWINDOWS_ASYNC: 488bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey if (!(devattr.device_cap_flags & IB_DEVICE_MEM_WINDOW)) { 489bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey dprintk("RPC: %s: MEMWINDOWS registration " 490bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey "specified but not supported by adapter, " 491bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey "using slower RPCRDMA_REGISTER\n", 492bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey __func__); 493bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey memreg = RPCRDMA_REGISTER; 494bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey } 495bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey break; 496bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_MTHCAFMR: 497bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey if (!ia->ri_id->device->alloc_fmr) { 498bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey#if RPCRDMA_PERSISTENT_REGISTRATION 499bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey dprintk("RPC: %s: MTHCAFMR registration " 500bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey "specified but not supported by adapter, " 501bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey "using riskier RPCRDMA_ALLPHYSICAL\n", 502bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey __func__); 503bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey memreg = RPCRDMA_ALLPHYSICAL; 504bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey#else 505bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey dprintk("RPC: %s: MTHCAFMR registration " 506bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey "specified but not supported by adapter, " 507bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey "using slower RPCRDMA_REGISTER\n", 508bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey __func__); 509bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey memreg = RPCRDMA_REGISTER; 510bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey#endif 511bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey } 512bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey break; 5133197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey case RPCRDMA_FRMR: 5143197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey /* Requires both frmr reg and local dma lkey */ 5153197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey if ((devattr.device_cap_flags & 5163197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey (IB_DEVICE_MEM_MGT_EXTENSIONS|IB_DEVICE_LOCAL_DMA_LKEY)) != 5173197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey (IB_DEVICE_MEM_MGT_EXTENSIONS|IB_DEVICE_LOCAL_DMA_LKEY)) { 5183197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey#if RPCRDMA_PERSISTENT_REGISTRATION 5193197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey dprintk("RPC: %s: FRMR registration " 5203197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey "specified but not supported by adapter, " 5213197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey "using riskier RPCRDMA_ALLPHYSICAL\n", 5223197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey __func__); 5233197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey memreg = RPCRDMA_ALLPHYSICAL; 5243197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey#else 5253197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey dprintk("RPC: %s: FRMR registration " 5263197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey "specified but not supported by adapter, " 5273197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey "using slower RPCRDMA_REGISTER\n", 5283197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey __func__); 5293197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey memreg = RPCRDMA_REGISTER; 5303197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey#endif 5313197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey } 5323197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey break; 533bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey } 534bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey 535bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey /* 536c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Optionally obtain an underlying physical identity mapping in 537c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * order to do a memory window-based bind. This base registration 538c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * is protected from remote access - that is enabled only by binding 539c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * for the specific bytes targeted during each RPC operation, and 540c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * revoked after the corresponding completion similar to a storage 541c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * adapter. 542c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 543bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey switch (memreg) { 544bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_BOUNCEBUFFERS: 545bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_REGISTER: 5463197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey case RPCRDMA_FRMR: 547bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey break; 548c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#if RPCRDMA_PERSISTENT_REGISTRATION 549bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_ALLPHYSICAL: 550bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey mem_priv = IB_ACCESS_LOCAL_WRITE | 551bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey IB_ACCESS_REMOTE_WRITE | 552bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey IB_ACCESS_REMOTE_READ; 553bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey goto register_setup; 554c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#endif 555bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_MEMWINDOWS_ASYNC: 556bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_MEMWINDOWS: 557bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey mem_priv = IB_ACCESS_LOCAL_WRITE | 558bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey IB_ACCESS_MW_BIND; 559bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey goto register_setup; 560bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_MTHCAFMR: 561bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey if (ia->ri_have_dma_lkey) 562c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 563bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey mem_priv = IB_ACCESS_LOCAL_WRITE; 564bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey register_setup: 565c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_bind_mem = ib_get_dma_mr(ia->ri_pd, mem_priv); 566c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(ia->ri_bind_mem)) { 567c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ printk(KERN_ALERT "%s: ib_get_dma_mr for " 568c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "phys register failed with %lX\n\t" 569c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "Will continue with degraded performance\n", 570c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, PTR_ERR(ia->ri_bind_mem)); 571c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ memreg = RPCRDMA_REGISTER; 572c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_bind_mem = NULL; 573c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 574bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey break; 575bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey default: 576bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey printk(KERN_ERR "%s: invalid memory registration mode %d\n", 577bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey __func__, memreg); 578bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey rc = -EINVAL; 579bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey goto out2; 580c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 581bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey dprintk("RPC: %s: memory registration strategy is %d\n", 582bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey __func__, memreg); 583c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 584c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Else will do memory reg/dereg for each chunk */ 585c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_memreg_strategy = memreg; 586c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 587c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 588c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out2: 589c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rdma_destroy_id(ia->ri_id); 590fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey ia->ri_id = NULL; 591c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out1: 592c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 593c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 594c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 595c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 596c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Clean up/close an IA. 597c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o if event handles and PD have been initialized, free them. 598c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o close the IA 599c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 600c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void 601c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ia_close(struct rpcrdma_ia *ia) 602c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 603c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 604c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 605c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: entering\n", __func__); 606c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ia->ri_bind_mem != NULL) { 607c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_dereg_mr(ia->ri_bind_mem); 608c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_dereg_mr returned %i\n", 609c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 610c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 611fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey if (ia->ri_id != NULL && !IS_ERR(ia->ri_id)) { 612fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey if (ia->ri_id->qp) 613fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey rdma_destroy_qp(ia->ri_id); 614fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey rdma_destroy_id(ia->ri_id); 615fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey ia->ri_id = NULL; 616fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey } 617c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ia->ri_pd != NULL && !IS_ERR(ia->ri_pd)) { 618c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_dealloc_pd(ia->ri_pd); 619c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_dealloc_pd returned %i\n", 620c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 621c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 622c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 623c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 624c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 625c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Create unconnected endpoint. 626c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 627c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 628c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia, 629c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_create_data_internal *cdata) 630c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 631c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_device_attr devattr; 6325d40a8a525c8165bafed233cf0f137e8d10d7e92Chuck Lever int rc, err; 633c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 634c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_query_device(ia->ri_id->device, &devattr); 635c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 636c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_query_device failed %d\n", 637c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 638c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 639c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 640c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 641c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* check provider's send/recv wr limits */ 642c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (cdata->max_requests > devattr.max_qp_wr) 643c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ cdata->max_requests = devattr.max_qp_wr; 644c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 645c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.event_handler = rpcrdma_qp_async_error_upcall; 646c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.qp_context = ep; 647c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* send_cq and recv_cq initialized below */ 648c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.srq = NULL; 649c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_wr = cdata->max_requests; 650c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 6513197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey case RPCRDMA_FRMR: 6523197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey /* Add room for frmr register and invalidate WRs */ 6533197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey ep->rep_attr.cap.max_send_wr *= 3; 6543197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey if (ep->rep_attr.cap.max_send_wr > devattr.max_qp_wr) 6553197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey return -EINVAL; 6563197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey break; 657c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS_ASYNC: 658c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS: 659c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Add room for mw_binds+unbinds - overkill! */ 660c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_wr++; 661c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_wr *= (2 * RPCRDMA_MAX_SEGS); 662c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_attr.cap.max_send_wr > devattr.max_qp_wr) 663c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return -EINVAL; 664c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 665c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 666c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 667c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 668c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_recv_wr = cdata->max_requests; 669c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_sge = (cdata->padding ? 4 : 2); 670c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_recv_sge = 1; 671c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_inline_data = 0; 672c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.sq_sig_type = IB_SIGNAL_REQ_WR; 673c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.qp_type = IB_QPT_RC; 674c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.port_num = ~0; 675c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 676c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: requested max: dtos: send %d recv %d; " 677c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "iovs: send %d recv %d\n", 678c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, 679c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_wr, 680c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_recv_wr, 681c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_sge, 682c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_recv_sge); 683c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 684c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* set trigger for requesting send completion */ 685c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_cqinit = ep->rep_attr.cap.max_send_wr/2 /* - 1*/; 686c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 687c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS_ASYNC: 688c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS: 689c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_cqinit -= RPCRDMA_MAX_SEGS; 690c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 691c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 692c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 693c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 694c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_cqinit <= 2) 695c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_cqinit = 0; 696c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ INIT_CQCOUNT(ep); 697c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_ia = ia; 698c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ init_waitqueue_head(&ep->rep_connect_wait); 699c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 700c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 701c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Create a single cq for receive dto and mw_bind (only ever 702c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * care about unbind, really). Send completions are suppressed. 703c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Use single threaded tasklet upcalls to maintain ordering. 704c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 705c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_cq = ib_create_cq(ia->ri_id->device, rpcrdma_cq_event_upcall, 706c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_cq_async_error_upcall, NULL, 707c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_recv_wr + 708c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_wr + 1, 0); 709c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(ep->rep_cq)) { 710c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(ep->rep_cq); 711c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_create_cq failed: %i\n", 712c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 713c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out1; 714c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 715c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 716c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_req_notify_cq(ep->rep_cq, IB_CQ_NEXT_COMP); 717c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 718c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_req_notify_cq failed: %i\n", 719c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 720c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out2; 721c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 722c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 723c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.send_cq = ep->rep_cq; 724c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.recv_cq = ep->rep_cq; 725c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 726c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Initialize cma parameters */ 727c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 728c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* RPC/RDMA does not use private data */ 729c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.private_data = NULL; 730c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.private_data_len = 0; 731c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 732c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Client offers RDMA Read but does not initiate */ 733b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker ep->rep_remote_cma.initiator_depth = 0; 734b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker if (ia->ri_memreg_strategy == RPCRDMA_BOUNCEBUFFERS) 735c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.responder_resources = 0; 736b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker else if (devattr.max_qp_rd_atom > 32) /* arbitrary but <= 255 */ 737b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker ep->rep_remote_cma.responder_resources = 32; 738b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker else 739c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.responder_resources = devattr.max_qp_rd_atom; 740c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 741c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.retry_count = 7; 742c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.flow_control = 0; 743c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.rnr_retry_count = 0; 744c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 745c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 746c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 747c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out2: 7485d40a8a525c8165bafed233cf0f137e8d10d7e92Chuck Lever err = ib_destroy_cq(ep->rep_cq); 7495d40a8a525c8165bafed233cf0f137e8d10d7e92Chuck Lever if (err) 7505d40a8a525c8165bafed233cf0f137e8d10d7e92Chuck Lever dprintk("RPC: %s: ib_destroy_cq returned %i\n", 7515d40a8a525c8165bafed233cf0f137e8d10d7e92Chuck Lever __func__, err); 752c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out1: 753c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 754c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 755c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 756c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 757c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * rpcrdma_ep_destroy 758c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 759c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Disconnect and destroy endpoint. After this, the only 760c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * valid operations on the ep are to free it (if dynamically 761c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * allocated) or re-create it. 762c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 763c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * The caller's error handling must be sure to not leak the endpoint 764c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * if this function fails. 765c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 766c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 767c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_destroy(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) 768c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 769c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 770c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 771c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: entering, connected is %d\n", 772c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, ep->rep_connected); 773c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 774c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ia->ri_id->qp) { 775c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_ep_disconnect(ep, ia); 776c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 777c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rpcrdma_ep_disconnect" 778c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " returned %i\n", __func__, rc); 779fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey rdma_destroy_qp(ia->ri_id); 780fee08caf943e8ed3446ce42fa085b5e7e5f08d92Tom Talpey ia->ri_id->qp = NULL; 781c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 782c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 783c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* padding - could be done in rpcrdma_buffer_destroy... */ 784c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_pad_mr) { 785c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_deregister_internal(ia, ep->rep_pad_mr, &ep->rep_pad); 786c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_pad_mr = NULL; 787c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 788c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 789c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_clean_cq(ep->rep_cq); 790c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_destroy_cq(ep->rep_cq); 791c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 792c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_destroy_cq returned %i\n", 793c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 794c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 795c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 796c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 797c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 798c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 799c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Connect unconnected endpoint. 800c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 801c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 802c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) 803c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 804c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rdma_cm_id *id; 805c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc = 0; 806c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int retry_count = 0; 807c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int reconnect = (ep->rep_connected != 0); 808c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 809c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (reconnect) { 810c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_xprt *xprt; 811c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\retry: 812c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_ep_disconnect(ep, ia); 813c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc && rc != -ENOTCONN) 814c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rpcrdma_ep_disconnect" 815c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " status %i\n", __func__, rc); 816c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_clean_cq(ep->rep_cq); 817c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 818c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ xprt = container_of(ia, struct rpcrdma_xprt, rx_ia); 819c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ id = rpcrdma_create_id(xprt, ia, 820c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (struct sockaddr *)&xprt->rx_data.addr); 821c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(id)) { 822c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(id); 823c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 824c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 825c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* TEMP TEMP TEMP - fail if new device: 826c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Deregister/remarshal *all* requests! 827c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Close and recreate adapter, pd, etc! 828c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Re-determine all attributes still sane! 829c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * More stuff I haven't thought of! 830c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Rrrgh! 831c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 832c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ia->ri_id->device != id->device) { 833c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ printk("RPC: %s: can't reconnect on " 834c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "different device!\n", __func__); 835c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rdma_destroy_id(id); 836c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = -ENETDOWN; 837c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 838c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 839c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* END TEMP */ 8401a954051b0cf79bd67e5f9db40333e3a9b1d05d2Tom Talpey rdma_destroy_qp(ia->ri_id); 841c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rdma_destroy_id(ia->ri_id); 842c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_id = id; 843c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 844c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 845c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rdma_create_qp(ia->ri_id, ia->ri_pd, &ep->rep_attr); 846c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 847c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_create_qp failed %i\n", 848c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 849c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 850c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 851c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 852c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* XXX Tavor device performs badly with 2K MTU! */ 853c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\if (strnicmp(ia->ri_id->device->dma_device->bus->name, "pci", 3) == 0) { 854c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct pci_dev *pcid = to_pci_dev(ia->ri_id->device->dma_device); 855c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (pcid->device == PCI_DEVICE_ID_MELLANOX_TAVOR && 856c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (pcid->vendor == PCI_VENDOR_ID_MELLANOX || 857c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ pcid->vendor == PCI_VENDOR_ID_TOPSPIN)) { 858c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_qp_attr attr = { 859c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ .path_mtu = IB_MTU_1024 860c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ }; 861c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_modify_qp(ia->ri_id->qp, &attr, IB_QP_PATH_MTU); 862c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 863c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 864c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 865c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = 0; 866c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 867c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rdma_connect(ia->ri_id, &ep->rep_remote_cma); 868c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 869c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_connect() failed with %i\n", 870c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 871c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 872c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 873c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 874c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (reconnect) 875c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 876c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 877c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wait_event_interruptible(ep->rep_connect_wait, ep->rep_connected != 0); 878c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 879c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 880c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Check state. A non-peer reject indicates no listener 881c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * (ECONNREFUSED), which may be a transient state. All 882c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * others indicate a transport condition which has already 883c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * undergone a best-effort. 884c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 885c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_connected == -ECONNREFUSED 886c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ && ++retry_count <= RDMA_CONNECT_RETRY_MAX) { 887c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: non-peer_reject, retry\n", __func__); 888c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto retry; 889c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 890c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_connected <= 0) { 891c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Sometimes, the only way to reliably connect to remote 892c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * CMs is to use same nonzero values for ORD and IRD. */ 893b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker if (retry_count++ <= RDMA_CONNECT_RETRY_MAX + 1 && 894b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker (ep->rep_remote_cma.responder_resources == 0 || 895b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker ep->rep_remote_cma.initiator_depth != 896b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker ep->rep_remote_cma.responder_resources)) { 897b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker if (ep->rep_remote_cma.responder_resources == 0) 898b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker ep->rep_remote_cma.responder_resources = 1; 899b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker ep->rep_remote_cma.initiator_depth = 900b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker ep->rep_remote_cma.responder_resources; 901c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto retry; 902b334eaabf4f92226d2df13c613888a507f03da99Tom Tucker } 903c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ep->rep_connected; 904c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } else { 905c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: connected\n", __func__); 906c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 907c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 908c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out: 909c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 910c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = rc; 911c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 912c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 913c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 914c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 915c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * rpcrdma_ep_disconnect 916c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 917c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * This is separate from destroy to facilitate the ability 918c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * to reconnect without recreating the endpoint. 919c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 920c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * This call is not reentrant, and must not be made in parallel 921c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * on the same endpoint. 922c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 923c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 924c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_disconnect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) 925c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 926c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 927c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 928c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_clean_cq(ep->rep_cq); 929c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rdma_disconnect(ia->ri_id); 930c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (!rc) { 931c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* returns without wait if not connected */ 932c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wait_event_interruptible(ep->rep_connect_wait, 933c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected != 1); 934c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: after wait, %sconnected\n", __func__, 935c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (ep->rep_connected == 1) ? "still " : "dis"); 936c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } else { 937c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_disconnect %i\n", __func__, rc); 938c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = rc; 939c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 940c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 941c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 942c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 943c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 944c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Initialize buffer memory 945c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 946c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 947c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_buffer_create(struct rpcrdma_buffer *buf, struct rpcrdma_ep *ep, 948c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia, struct rpcrdma_create_data_internal *cdata) 949c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 950c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ char *p; 951c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ size_t len; 952c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int i, rc; 9538d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_mw *r; 954c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 955c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_max_requests = cdata->max_requests; 956c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_init(&buf->rb_lock); 957c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ atomic_set(&buf->rb_credits, 1); 958c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 959c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Need to allocate: 960c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 1. arrays for send and recv pointers 961c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 2. arrays of struct rpcrdma_req to fill in pointers 962c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 3. array of struct rpcrdma_rep for replies 963c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 4. padding, if any 9643197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey * 5. mw's, fmr's or frmr's, if any 965c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Send/recv buffers in req/rep need to be registered 966c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 967c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 968c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len = buf->rb_max_requests * 969c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (sizeof(struct rpcrdma_req *) + sizeof(struct rpcrdma_rep *)); 970c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len += cdata->padding; 971c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 9723197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey case RPCRDMA_FRMR: 9733197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey len += buf->rb_max_requests * RPCRDMA_MAX_SEGS * 9743197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey sizeof(struct rpcrdma_mw); 9753197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey break; 976c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 977c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* TBD we are perhaps overallocating here */ 978c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len += (buf->rb_max_requests + 1) * RPCRDMA_MAX_SEGS * 979c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ sizeof(struct rpcrdma_mw); 980c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 981c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS_ASYNC: 982c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS: 983c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len += (buf->rb_max_requests + 1) * RPCRDMA_MAX_SEGS * 984c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ sizeof(struct rpcrdma_mw); 985c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 986c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 987c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 988c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 989c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 990c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* allocate 1, 4 and 5 in one shot */ 991c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ p = kzalloc(len, GFP_KERNEL); 992c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (p == NULL) { 993c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: req_t/rep_t/pad kzalloc(%zd) failed\n", 994c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, len); 995c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = -ENOMEM; 996c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 997c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 998c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_pool = p; /* for freeing it later */ 999c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1000c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_send_bufs = (struct rpcrdma_req **) p; 1001c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ p = (char *) &buf->rb_send_bufs[buf->rb_max_requests]; 1002c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_recv_bufs = (struct rpcrdma_rep **) p; 1003c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ p = (char *) &buf->rb_recv_bufs[buf->rb_max_requests]; 1004c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1005c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 1006c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Register the zeroed pad buffer, if any. 1007c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1008c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (cdata->padding) { 1009c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_register_internal(ia, p, cdata->padding, 1010c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &ep->rep_pad_mr, &ep->rep_pad); 1011c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1012c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1013c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1014c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ p += cdata->padding; 1015c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1016c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 1017c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Allocate the fmr's, or mw's for mw_bind chunk registration. 1018c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * We "cycle" the mw's in order to minimize rkey reuse, 1019c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * and also reduce unbind-to-bind collision. 1020c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1021c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ INIT_LIST_HEAD(&buf->rb_mws); 10228d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey r = (struct rpcrdma_mw *)p; 1023c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 10243197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey case RPCRDMA_FRMR: 10253197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey for (i = buf->rb_max_requests * RPCRDMA_MAX_SEGS; i; i--) { 10263197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey r->r.frmr.fr_mr = ib_alloc_fast_reg_mr(ia->ri_pd, 10273197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey RPCRDMA_MAX_SEGS); 10283197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey if (IS_ERR(r->r.frmr.fr_mr)) { 10293197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey rc = PTR_ERR(r->r.frmr.fr_mr); 10303197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey dprintk("RPC: %s: ib_alloc_fast_reg_mr" 10313197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey " failed %i\n", __func__, rc); 10323197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey goto out; 10333197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey } 10343197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey r->r.frmr.fr_pgl = 10353197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey ib_alloc_fast_reg_page_list(ia->ri_id->device, 10363197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey RPCRDMA_MAX_SEGS); 10373197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey if (IS_ERR(r->r.frmr.fr_pgl)) { 10383197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey rc = PTR_ERR(r->r.frmr.fr_pgl); 10393197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey dprintk("RPC: %s: " 10403197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey "ib_alloc_fast_reg_page_list " 10413197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey "failed %i\n", __func__, rc); 10423197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey goto out; 10433197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey } 10443197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey list_add(&r->mw_list, &buf->rb_mws); 10453197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey ++r; 10463197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey } 10473197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey break; 1048c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 1049c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* TBD we are perhaps overallocating here */ 1050c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ for (i = (buf->rb_max_requests+1) * RPCRDMA_MAX_SEGS; i; i--) { 10518d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey static struct ib_fmr_attr fa = 10528d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey { RPCRDMA_MAX_DATA_SEGS, 1, PAGE_SHIFT }; 1053c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ r->r.fmr = ib_alloc_fmr(ia->ri_pd, 1054c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ, 1055c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &fa); 1056c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(r->r.fmr)) { 1057c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(r->r.fmr); 1058c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_alloc_fmr" 1059c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " failed %i\n", __func__, rc); 1060c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1061c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1062c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_add(&r->mw_list, &buf->rb_mws); 1063c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ++r; 1064c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1065c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1066c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS_ASYNC: 1067c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS: 1068c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Allocate one extra request's worth, for full cycling */ 1069c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ for (i = (buf->rb_max_requests+1) * RPCRDMA_MAX_SEGS; i; i--) { 1070c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ r->r.mw = ib_alloc_mw(ia->ri_pd); 1071c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(r->r.mw)) { 1072c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(r->r.mw); 1073c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_alloc_mw" 1074c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " failed %i\n", __func__, rc); 1075c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1076c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1077c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_add(&r->mw_list, &buf->rb_mws); 1078c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ++r; 1079c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1080c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1081c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 1082c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1083c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1084c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1085c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 1086c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Allocate/init the request/reply buffers. Doing this 1087c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * using kmalloc for now -- one for each buf. 1088c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1089c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ for (i = 0; i < buf->rb_max_requests; i++) { 1090c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_req *req; 1091c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep *rep; 1092c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1093c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len = cdata->inline_wsize + sizeof(struct rpcrdma_req); 1094c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* RPC layer requests *double* size + 1K RPC_SLACK_SPACE! */ 1095c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Typical ~2400b, so rounding up saves work later */ 1096c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (len < 4096) 1097c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len = 4096; 1098c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req = kmalloc(len, GFP_KERNEL); 1099c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (req == NULL) { 1100c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: request buffer %d alloc" 1101c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " failed\n", __func__, i); 1102c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = -ENOMEM; 1103c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1104c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1105c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ memset(req, 0, sizeof(struct rpcrdma_req)); 1106c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_send_bufs[i] = req; 1107c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_send_bufs[i]->rl_buffer = buf; 1108c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1109c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_register_internal(ia, req->rl_base, 1110c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len - offsetof(struct rpcrdma_req, rl_base), 1111c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_send_bufs[i]->rl_handle, 1112c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_send_bufs[i]->rl_iov); 1113c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1114c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1115c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1116c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_send_bufs[i]->rl_size = len-sizeof(struct rpcrdma_req); 1117c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1118c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len = cdata->inline_rsize + sizeof(struct rpcrdma_rep); 1119c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep = kmalloc(len, GFP_KERNEL); 1120c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rep == NULL) { 1121c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: reply buffer %d alloc failed\n", 1122c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, i); 1123c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = -ENOMEM; 1124c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1125c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1126c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ memset(rep, 0, sizeof(struct rpcrdma_rep)); 1127c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_recv_bufs[i] = rep; 1128c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_recv_bufs[i]->rr_buffer = buf; 1129c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ init_waitqueue_head(&rep->rr_unbind); 1130c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1131c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_register_internal(ia, rep->rr_base, 1132c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len - offsetof(struct rpcrdma_rep, rr_base), 1133c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_recv_bufs[i]->rr_handle, 1134c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_recv_bufs[i]->rr_iov); 1135c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1136c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1137c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1138c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1139c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: max_requests %d\n", 1140c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, buf->rb_max_requests); 1141c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* done */ 1142c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 1143c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out: 1144c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_buffer_destroy(buf); 1145c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 1146c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1147c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1148c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1149c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Unregister and destroy buffer memory. Need to deal with 1150c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * partial initialization, so it's callable from failed create. 1151c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Must be called before destroying endpoint, as registrations 1152c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * reference it. 1153c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1154c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void 1155c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf) 1156c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1157c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc, i; 1158c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = rdmab_to_ia(buf); 11598d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_mw *r; 1160c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1161c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* clean up in reverse order from create 1162c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 1. recv mr memory (mr free, then kfree) 1163c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 1a. bind mw memory 1164c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 2. send mr memory (mr free, then kfree) 1165c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 3. padding (if any) [moved to rpcrdma_ep_destroy] 1166c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 4. arrays 1167c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1168c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: entering\n", __func__); 1169c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1170c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ for (i = 0; i < buf->rb_max_requests; i++) { 1171c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (buf->rb_recv_bufs && buf->rb_recv_bufs[i]) { 1172c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_deregister_internal(ia, 1173c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_recv_bufs[i]->rr_handle, 1174c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_recv_bufs[i]->rr_iov); 1175c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ kfree(buf->rb_recv_bufs[i]); 1176c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1177c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (buf->rb_send_bufs && buf->rb_send_bufs[i]) { 1178c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ while (!list_empty(&buf->rb_mws)) { 1179c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ r = list_entry(buf->rb_mws.next, 1180c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_mw, mw_list); 1181c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_del(&r->mw_list); 1182c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 11833197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey case RPCRDMA_FRMR: 11843197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey rc = ib_dereg_mr(r->r.frmr.fr_mr); 11853197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey if (rc) 11863197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey dprintk("RPC: %s:" 11873197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey " ib_dereg_mr" 11883197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey " failed %i\n", 11893197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey __func__, rc); 11903197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey ib_free_fast_reg_page_list(r->r.frmr.fr_pgl); 11913197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey break; 1192c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 1193c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_dealloc_fmr(r->r.fmr); 1194c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1195c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s:" 1196c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " ib_dealloc_fmr" 1197c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " failed %i\n", 1198c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 1199c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1200c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS_ASYNC: 1201c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS: 1202c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_dealloc_mw(r->r.mw); 1203c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1204c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s:" 1205c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " ib_dealloc_mw" 1206c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " failed %i\n", 1207c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 1208c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1209c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 1210c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1211c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1212c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1213c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_deregister_internal(ia, 1214c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_send_bufs[i]->rl_handle, 1215c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_send_bufs[i]->rl_iov); 1216c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ kfree(buf->rb_send_bufs[i]); 1217c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1218c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1219c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1220c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ kfree(buf->rb_pool); 1221c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1222c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1223c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1224c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Get a set of request/reply buffers. 1225c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 1226c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Reply buffer (if needed) is attached to send buffer upon return. 1227c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Rule: 1228c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * rb_send_index and rb_recv_index MUST always be pointing to the 1229c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * *next* available buffer (non-NULL). They are incremented after 1230c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * removing buffers, and decremented *before* returning them. 1231c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1232c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\struct rpcrdma_req * 1233c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_buffer_get(struct rpcrdma_buffer *buffers) 1234c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1235c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_req *req; 1236c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 12378d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int i; 12388d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_mw *r; 1239c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1240c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&buffers->rb_lock, flags); 1241c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (buffers->rb_send_index == buffers->rb_max_requests) { 1242c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&buffers->rb_lock, flags); 1243c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: out of request buffers\n", __func__); 1244c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return ((struct rpcrdma_req *)NULL); 1245c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1246c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1247c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req = buffers->rb_send_bufs[buffers->rb_send_index]; 1248c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (buffers->rb_send_index < buffers->rb_recv_index) { 1249c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: %d extra receives outstanding (ok)\n", 1250c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, 1251c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_recv_index - buffers->rb_send_index); 1252c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply = NULL; 1253c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } else { 1254c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply = buffers->rb_recv_bufs[buffers->rb_recv_index]; 1255c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_recv_bufs[buffers->rb_recv_index++] = NULL; 1256c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1257c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_send_bufs[buffers->rb_send_index++] = NULL; 1258c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (!list_empty(&buffers->rb_mws)) { 12598d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey i = RPCRDMA_MAX_SEGS - 1; 1260c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ do { 1261c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ r = list_entry(buffers->rb_mws.next, 1262c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_mw, mw_list); 1263c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_del(&r->mw_list); 1264c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_segments[i].mr_chunk.rl_mw = r; 1265c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } while (--i >= 0); 1266c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1267c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&buffers->rb_lock, flags); 1268c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return req; 1269c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1270c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1271c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1272c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Put request/reply buffers back into pool. 1273c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Pre-decrement counter/array index. 1274c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1275c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void 1276c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_buffer_put(struct rpcrdma_req *req) 1277c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1278c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_buffer *buffers = req->rl_buffer; 1279c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = rdmab_to_ia(buffers); 1280c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int i; 1281c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 1282c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1283c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ BUG_ON(req->rl_nchunks != 0); 1284c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&buffers->rb_lock, flags); 1285c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_send_bufs[--buffers->rb_send_index] = req; 1286c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_niovs = 0; 1287c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (req->rl_reply) { 1288c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_recv_bufs[--buffers->rb_recv_index] = req->rl_reply; 1289c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ init_waitqueue_head(&req->rl_reply->rr_unbind); 1290c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply->rr_func = NULL; 1291c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply = NULL; 1292c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1293c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 12943197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey case RPCRDMA_FRMR: 1295c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 1296c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS_ASYNC: 1297c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS: 1298c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 1299c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Cycle mw's back in reverse order, and "spin" them. 1300c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * This delays and scrambles reuse as much as possible. 1301c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1302c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ i = 1; 1303c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ do { 1304c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_mw **mw; 1305c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ mw = &req->rl_segments[i].mr_chunk.rl_mw; 1306c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_add_tail(&(*mw)->mw_list, &buffers->rb_mws); 1307c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ *mw = NULL; 1308c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } while (++i < RPCRDMA_MAX_SEGS); 1309c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_add_tail(&req->rl_segments[0].mr_chunk.rl_mw->mw_list, 1310c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buffers->rb_mws); 1311c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_segments[0].mr_chunk.rl_mw = NULL; 1312c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1313c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 1314c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1315c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1316c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&buffers->rb_lock, flags); 1317c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1318c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1319c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1320c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Recover reply buffers from pool. 1321c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * This happens when recovering from error conditions. 1322c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Post-increment counter/array index. 1323c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1324c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void 1325c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_recv_buffer_get(struct rpcrdma_req *req) 1326c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1327c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_buffer *buffers = req->rl_buffer; 1328c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 1329c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1330c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (req->rl_iov.length == 0) /* special case xprt_rdma_allocate() */ 1331c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers = ((struct rpcrdma_req *) buffers)->rl_buffer; 1332c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&buffers->rb_lock, flags); 1333c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (buffers->rb_recv_index < buffers->rb_max_requests) { 1334c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply = buffers->rb_recv_bufs[buffers->rb_recv_index]; 1335c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_recv_bufs[buffers->rb_recv_index++] = NULL; 1336c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1337c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&buffers->rb_lock, flags); 1338c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1339c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1340c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1341c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Put reply buffers back into pool when not attached to 1342c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * request. This happens in error conditions, and when 1343c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * aborting unbinds. Pre-decrement counter/array index. 1344c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1345c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void 1346c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep) 1347c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1348c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_buffer *buffers = rep->rr_buffer; 1349c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 1350c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1351c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_func = NULL; 1352c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&buffers->rb_lock, flags); 1353c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_recv_bufs[--buffers->rb_recv_index] = rep; 1354c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&buffers->rb_lock, flags); 1355c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1356c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1357c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1358c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Wrappers for internal-use kmalloc memory registration, used by buffer code. 1359c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1360c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1361c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1362c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_register_internal(struct rpcrdma_ia *ia, void *va, int len, 1363c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_mr **mrp, struct ib_sge *iov) 1364c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1365c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_phys_buf ipb; 1366c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_mr *mr; 1367c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 1368c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1369c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 1370c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * All memory passed here was kmalloc'ed, therefore phys-contiguous. 1371c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1372c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ iov->addr = ib_dma_map_single(ia->ri_id->device, 1373c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ va, len, DMA_BIDIRECTIONAL); 1374c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ iov->length = len; 1375c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1376bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey if (ia->ri_have_dma_lkey) { 1377bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey *mrp = NULL; 1378bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey iov->lkey = ia->ri_dma_lkey; 1379bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey return 0; 1380bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey } else if (ia->ri_bind_mem != NULL) { 1381c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ *mrp = NULL; 1382c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ iov->lkey = ia->ri_bind_mem->lkey; 1383c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 1384c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1385c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1386c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ipb.addr = iov->addr; 1387c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ipb.size = iov->length; 1388c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ mr = ib_reg_phys_mr(ia->ri_pd, &ipb, 1, 1389c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ IB_ACCESS_LOCAL_WRITE, &iov->addr); 1390c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1391c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: phys convert: 0x%llx " 1392c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "registered 0x%llx length %d\n", 1393a56daeb7d5c9a05b1cb52ae4bcca05fb6545656eAndrew Morton __func__, (unsigned long long)ipb.addr, 1394a56daeb7d5c9a05b1cb52ae4bcca05fb6545656eAndrew Morton (unsigned long long)iov->addr, len); 1395c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1396c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(mr)) { 1397c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ *mrp = NULL; 1398c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(mr); 1399c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: failed with %i\n", __func__, rc); 1400c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } else { 1401c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ *mrp = mr; 1402c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ iov->lkey = mr->lkey; 1403c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = 0; 1404c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1405c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1406c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 1407c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1408c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1409c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1410c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_deregister_internal(struct rpcrdma_ia *ia, 1411c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_mr *mr, struct ib_sge *iov) 1412c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1413c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 1414c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1415c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_unmap_single(ia->ri_id->device, 1416c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ iov->addr, iov->length, DMA_BIDIRECTIONAL); 1417c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1418c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (NULL == mr) 1419c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 1420c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1421c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_dereg_mr(mr); 1422c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1423c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_dereg_mr failed %i\n", __func__, rc); 1424c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 1425c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1426c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1427c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1428c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Wrappers for chunk registration, shared by read/write chunk code. 1429c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1430c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1431c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 1432c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_map_one(struct rpcrdma_ia *ia, struct rpcrdma_mr_seg *seg, int writing) 1433c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1434c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dir = writing ? DMA_FROM_DEVICE : DMA_TO_DEVICE; 1435c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dmalen = seg->mr_len; 1436c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (seg->mr_page) 1437c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dma = ib_dma_map_page(ia->ri_id->device, 1438c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_page, offset_in_page(seg->mr_offset), 1439c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dmalen, seg->mr_dir); 1440c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ else 1441c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dma = ib_dma_map_single(ia->ri_id->device, 1442c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_offset, 1443c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dmalen, seg->mr_dir); 1444c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1445c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1446c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 1447c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_unmap_one(struct rpcrdma_ia *ia, struct rpcrdma_mr_seg *seg) 1448c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1449c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (seg->mr_page) 1450c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_unmap_page(ia->ri_id->device, 1451c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dma, seg->mr_dmalen, seg->mr_dir); 1452c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ else 1453c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_unmap_single(ia->ri_id->device, 1454c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dma, seg->mr_dmalen, seg->mr_dir); 1455c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1456c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 14578d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeystatic int 14583197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpeyrpcrdma_register_frmr_external(struct rpcrdma_mr_seg *seg, 14593197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey int *nsegs, int writing, struct rpcrdma_ia *ia, 14603197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey struct rpcrdma_xprt *r_xprt) 14613197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey{ 14623197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey struct rpcrdma_mr_seg *seg1 = seg; 14633197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey struct ib_send_wr frmr_wr, *bad_wr; 14643197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey u8 key; 14653197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey int len, pageoff; 14663197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey int i, rc; 14673197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 14683197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey pageoff = offset_in_page(seg1->mr_offset); 14693197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey seg1->mr_offset -= pageoff; /* start of page */ 14703197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey seg1->mr_len += pageoff; 14713197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey len = -pageoff; 14723197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey if (*nsegs > RPCRDMA_MAX_DATA_SEGS) 14733197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey *nsegs = RPCRDMA_MAX_DATA_SEGS; 14743197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey for (i = 0; i < *nsegs;) { 14753197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey rpcrdma_map_one(ia, seg, writing); 14763197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey seg1->mr_chunk.rl_mw->r.frmr.fr_pgl->page_list[i] = seg->mr_dma; 14773197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey len += seg->mr_len; 14783197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey ++seg; 14793197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey ++i; 14803197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey /* Check for holes */ 14813197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey if ((i < *nsegs && offset_in_page(seg->mr_offset)) || 14823197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len)) 14833197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey break; 14843197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey } 14853197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey dprintk("RPC: %s: Using frmr %p to map %d segments\n", 14863197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey __func__, seg1->mr_chunk.rl_mw, i); 14873197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 14883197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey /* Bump the key */ 14893197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey key = (u8)(seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey & 0x000000FF); 14903197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey ib_update_fast_reg_key(seg1->mr_chunk.rl_mw->r.frmr.fr_mr, ++key); 14913197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 14923197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey /* Prepare FRMR WR */ 14933197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey memset(&frmr_wr, 0, sizeof frmr_wr); 14943197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey frmr_wr.opcode = IB_WR_FAST_REG_MR; 14953197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey frmr_wr.send_flags = 0; /* unsignaled */ 14963197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey frmr_wr.wr.fast_reg.iova_start = (unsigned long)seg1->mr_dma; 14973197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey frmr_wr.wr.fast_reg.page_list = seg1->mr_chunk.rl_mw->r.frmr.fr_pgl; 14983197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey frmr_wr.wr.fast_reg.page_list_len = i; 14993197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey frmr_wr.wr.fast_reg.page_shift = PAGE_SHIFT; 15003197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey frmr_wr.wr.fast_reg.length = i << PAGE_SHIFT; 15013197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey frmr_wr.wr.fast_reg.access_flags = (writing ? 15023197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey IB_ACCESS_REMOTE_WRITE : IB_ACCESS_REMOTE_READ); 15033197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey frmr_wr.wr.fast_reg.rkey = seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey; 15043197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey DECR_CQCOUNT(&r_xprt->rx_ep); 15053197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 15063197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey rc = ib_post_send(ia->ri_id->qp, &frmr_wr, &bad_wr); 15073197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 15083197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey if (rc) { 15093197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey dprintk("RPC: %s: failed ib_post_send for register," 15103197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey " status %i\n", __func__, rc); 15113197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey while (i--) 15123197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey rpcrdma_unmap_one(ia, --seg); 15133197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey } else { 15143197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey seg1->mr_rkey = seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey; 15153197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey seg1->mr_base = seg1->mr_dma + pageoff; 15163197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey seg1->mr_nsegs = i; 15173197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey seg1->mr_len = len; 15183197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey } 15193197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey *nsegs = i; 15203197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey return rc; 15213197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey} 15223197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 15233197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpeystatic int 15243197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpeyrpcrdma_deregister_frmr_external(struct rpcrdma_mr_seg *seg, 15253197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey struct rpcrdma_ia *ia, struct rpcrdma_xprt *r_xprt) 15263197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey{ 15273197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey struct rpcrdma_mr_seg *seg1 = seg; 15283197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey struct ib_send_wr invalidate_wr, *bad_wr; 15293197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey int rc; 15303197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 15313197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey while (seg1->mr_nsegs--) 15323197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey rpcrdma_unmap_one(ia, seg++); 15333197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 15343197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey memset(&invalidate_wr, 0, sizeof invalidate_wr); 15353197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey invalidate_wr.opcode = IB_WR_LOCAL_INV; 15363197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey invalidate_wr.send_flags = 0; /* unsignaled */ 15373197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey invalidate_wr.ex.invalidate_rkey = seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey; 15383197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey DECR_CQCOUNT(&r_xprt->rx_ep); 15393197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 15403197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey rc = ib_post_send(ia->ri_id->qp, &invalidate_wr, &bad_wr); 15413197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey if (rc) 15423197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey dprintk("RPC: %s: failed ib_post_send for invalidate," 15433197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey " status %i\n", __func__, rc); 15443197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey return rc; 15453197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey} 15463197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 15473197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpeystatic int 15488d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeyrpcrdma_register_fmr_external(struct rpcrdma_mr_seg *seg, 15498d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int *nsegs, int writing, struct rpcrdma_ia *ia) 15508d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey{ 15518d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_mr_seg *seg1 = seg; 15528d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey u64 physaddrs[RPCRDMA_MAX_DATA_SEGS]; 15538d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int len, pageoff, i, rc; 15548d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 15558d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey pageoff = offset_in_page(seg1->mr_offset); 15568d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_offset -= pageoff; /* start of page */ 15578d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_len += pageoff; 15588d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey len = -pageoff; 15598d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (*nsegs > RPCRDMA_MAX_DATA_SEGS) 15608d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey *nsegs = RPCRDMA_MAX_DATA_SEGS; 15618d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey for (i = 0; i < *nsegs;) { 15628d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_map_one(ia, seg, writing); 15638d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey physaddrs[i] = seg->mr_dma; 15648d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey len += seg->mr_len; 15658d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey ++seg; 15668d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey ++i; 15678d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey /* Check for holes */ 15688d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if ((i < *nsegs && offset_in_page(seg->mr_offset)) || 15698d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len)) 15708d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey break; 15718d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } 15728d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = ib_map_phys_fmr(seg1->mr_chunk.rl_mw->r.fmr, 15738d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey physaddrs, i, seg1->mr_dma); 15748d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (rc) { 15758d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey dprintk("RPC: %s: failed ib_map_phys_fmr " 15768d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey "%u@0x%llx+%i (%d)... status %i\n", __func__, 15778d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey len, (unsigned long long)seg1->mr_dma, 15788d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey pageoff, i, rc); 15798d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey while (i--) 15808d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_unmap_one(ia, --seg); 15818d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } else { 15828d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_rkey = seg1->mr_chunk.rl_mw->r.fmr->rkey; 15838d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_base = seg1->mr_dma + pageoff; 15848d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_nsegs = i; 15858d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_len = len; 15868d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } 15878d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey *nsegs = i; 15888d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey return rc; 15898d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey} 15908d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 15918d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeystatic int 15928d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeyrpcrdma_deregister_fmr_external(struct rpcrdma_mr_seg *seg, 15938d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_ia *ia) 15948d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey{ 15958d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_mr_seg *seg1 = seg; 15968d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey LIST_HEAD(l); 15978d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int rc; 15988d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 15998d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey list_add(&seg1->mr_chunk.rl_mw->r.fmr->list, &l); 16008d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = ib_unmap_fmr(&l); 16018d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey while (seg1->mr_nsegs--) 16028d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_unmap_one(ia, seg++); 16038d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (rc) 16048d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey dprintk("RPC: %s: failed ib_unmap_fmr," 16058d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey " status %i\n", __func__, rc); 16068d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey return rc; 16078d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey} 16088d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 16098d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeystatic int 16108d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeyrpcrdma_register_memwin_external(struct rpcrdma_mr_seg *seg, 16118d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int *nsegs, int writing, struct rpcrdma_ia *ia, 16128d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_xprt *r_xprt) 16138d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey{ 16148d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int mem_priv = (writing ? IB_ACCESS_REMOTE_WRITE : 16158d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey IB_ACCESS_REMOTE_READ); 16168d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct ib_mw_bind param; 16178d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int rc; 16188d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 16198d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey *nsegs = 1; 16208d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_map_one(ia, seg, writing); 16218d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.mr = ia->ri_bind_mem; 16228d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.wr_id = 0ULL; /* no send cookie */ 16238d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.addr = seg->mr_dma; 16248d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.length = seg->mr_len; 16258d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.send_flags = 0; 16268d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.mw_access_flags = mem_priv; 16278d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 16288d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey DECR_CQCOUNT(&r_xprt->rx_ep); 16298d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = ib_bind_mw(ia->ri_id->qp, seg->mr_chunk.rl_mw->r.mw, ¶m); 16308d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (rc) { 16318d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey dprintk("RPC: %s: failed ib_bind_mw " 16328d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey "%u@0x%llx status %i\n", 16338d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey __func__, seg->mr_len, 16348d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey (unsigned long long)seg->mr_dma, rc); 16358d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_unmap_one(ia, seg); 16368d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } else { 16378d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg->mr_rkey = seg->mr_chunk.rl_mw->r.mw->rkey; 16388d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg->mr_base = param.addr; 16398d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg->mr_nsegs = 1; 16408d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } 16418d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey return rc; 16428d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey} 16438d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 16448d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeystatic int 16458d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeyrpcrdma_deregister_memwin_external(struct rpcrdma_mr_seg *seg, 16468d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_ia *ia, 16478d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_xprt *r_xprt, void **r) 16488d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey{ 16498d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct ib_mw_bind param; 16508d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey LIST_HEAD(l); 16518d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int rc; 16528d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 16538d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey BUG_ON(seg->mr_nsegs != 1); 16548d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.mr = ia->ri_bind_mem; 16558d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.addr = 0ULL; /* unbind */ 16568d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.length = 0; 16578d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.mw_access_flags = 0; 16588d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (*r) { 16598d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.wr_id = (u64) (unsigned long) *r; 16608d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.send_flags = IB_SEND_SIGNALED; 16618d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey INIT_CQCOUNT(&r_xprt->rx_ep); 16628d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } else { 16638d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.wr_id = 0ULL; 16648d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.send_flags = 0; 16658d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey DECR_CQCOUNT(&r_xprt->rx_ep); 16668d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } 16678d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = ib_bind_mw(ia->ri_id->qp, seg->mr_chunk.rl_mw->r.mw, ¶m); 16688d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_unmap_one(ia, seg); 16698d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (rc) 16708d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey dprintk("RPC: %s: failed ib_(un)bind_mw," 16718d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey " status %i\n", __func__, rc); 16728d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey else 16738d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey *r = NULL; /* will upcall on completion */ 16748d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey return rc; 16758d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey} 16768d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 16778d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeystatic int 16788d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeyrpcrdma_register_default_external(struct rpcrdma_mr_seg *seg, 16798d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int *nsegs, int writing, struct rpcrdma_ia *ia) 16808d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey{ 16818d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int mem_priv = (writing ? IB_ACCESS_REMOTE_WRITE : 16828d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey IB_ACCESS_REMOTE_READ); 16838d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_mr_seg *seg1 = seg; 16848d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct ib_phys_buf ipb[RPCRDMA_MAX_DATA_SEGS]; 16858d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int len, i, rc = 0; 16868d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 16878d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (*nsegs > RPCRDMA_MAX_DATA_SEGS) 16888d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey *nsegs = RPCRDMA_MAX_DATA_SEGS; 16898d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey for (len = 0, i = 0; i < *nsegs;) { 16908d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_map_one(ia, seg, writing); 16918d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey ipb[i].addr = seg->mr_dma; 16928d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey ipb[i].size = seg->mr_len; 16938d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey len += seg->mr_len; 16948d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey ++seg; 16958d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey ++i; 16968d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey /* Check for holes */ 16978d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if ((i < *nsegs && offset_in_page(seg->mr_offset)) || 16988d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey offset_in_page((seg-1)->mr_offset+(seg-1)->mr_len)) 16998d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey break; 17008d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } 17018d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_base = seg1->mr_dma; 17028d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_chunk.rl_mr = ib_reg_phys_mr(ia->ri_pd, 17038d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey ipb, i, mem_priv, &seg1->mr_base); 17048d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (IS_ERR(seg1->mr_chunk.rl_mr)) { 17058d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = PTR_ERR(seg1->mr_chunk.rl_mr); 17068d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey dprintk("RPC: %s: failed ib_reg_phys_mr " 17078d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey "%u@0x%llx (%d)... status %i\n", 17088d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey __func__, len, 17098d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey (unsigned long long)seg1->mr_dma, i, rc); 17108d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey while (i--) 17118d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_unmap_one(ia, --seg); 17128d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } else { 17138d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_rkey = seg1->mr_chunk.rl_mr->rkey; 17148d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_nsegs = i; 17158d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_len = len; 17168d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } 17178d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey *nsegs = i; 17188d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey return rc; 17198d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey} 17208d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 17218d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeystatic int 17228d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeyrpcrdma_deregister_default_external(struct rpcrdma_mr_seg *seg, 17238d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_ia *ia) 17248d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey{ 17258d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_mr_seg *seg1 = seg; 17268d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int rc; 17278d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 17288d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = ib_dereg_mr(seg1->mr_chunk.rl_mr); 17298d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_chunk.rl_mr = NULL; 17308d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey while (seg1->mr_nsegs--) 17318d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_unmap_one(ia, seg++); 17328d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (rc) 17338d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey dprintk("RPC: %s: failed ib_dereg_mr," 17348d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey " status %i\n", __func__, rc); 17358d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey return rc; 17368d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey} 17378d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 1738c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1739c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_register_external(struct rpcrdma_mr_seg *seg, 1740c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int nsegs, int writing, struct rpcrdma_xprt *r_xprt) 1741c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1742c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = &r_xprt->rx_ia; 1743c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc = 0; 1744c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1745c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 1746c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1747c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#if RPCRDMA_PERSISTENT_REGISTRATION 1748c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_ALLPHYSICAL: 1749c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_map_one(ia, seg, writing); 1750c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_rkey = ia->ri_bind_mem->rkey; 1751c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_base = seg->mr_dma; 1752c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_nsegs = 1; 1753c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ nsegs = 1; 1754c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1755c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#endif 1756c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 17573197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey /* Registration using frmr registration */ 17583197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey case RPCRDMA_FRMR: 17593197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey rc = rpcrdma_register_frmr_external(seg, &nsegs, writing, ia, r_xprt); 17603197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey break; 17613197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 17628d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey /* Registration using fmr memory registration */ 1763c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 17648d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = rpcrdma_register_fmr_external(seg, &nsegs, writing, ia); 1765c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1766c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1767c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Registration using memory windows */ 1768c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS_ASYNC: 1769c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS: 17708d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = rpcrdma_register_memwin_external(seg, &nsegs, writing, ia, r_xprt); 1771c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1772c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1773c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Default registration each time */ 1774c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 17758d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = rpcrdma_register_default_external(seg, &nsegs, writing, ia); 1776c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1777c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1778c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1779c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return -1; 1780c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1781c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return nsegs; 1782c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1783c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1784c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1785c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_deregister_external(struct rpcrdma_mr_seg *seg, 1786c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_xprt *r_xprt, void *r) 1787c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1788c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = &r_xprt->rx_ia; 1789c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int nsegs = seg->mr_nsegs, rc; 1790c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1791c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 1792c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1793c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#if RPCRDMA_PERSISTENT_REGISTRATION 1794c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_ALLPHYSICAL: 1795c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ BUG_ON(nsegs != 1); 1796c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_unmap_one(ia, seg); 1797c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = 0; 1798c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1799c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#endif 1800c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 18013197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey case RPCRDMA_FRMR: 18023197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey rc = rpcrdma_deregister_frmr_external(seg, ia, r_xprt); 18033197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey break; 18043197d309f5fb042499b2c4c8f2fcb67372df5201Tom Talpey 1805c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 18068d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = rpcrdma_deregister_fmr_external(seg, ia); 1807c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1808c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1809c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS_ASYNC: 1810c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS: 18118d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = rpcrdma_deregister_memwin_external(seg, ia, r_xprt, &r); 1812c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1813c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1814c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 18158d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = rpcrdma_deregister_default_external(seg, ia); 1816c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1817c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1818c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (r) { 1819c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep *rep = r; 1820c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ void (*func)(struct rpcrdma_rep *) = rep->rr_func; 1821c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_func = NULL; 1822c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ func(rep); /* dereg done, callback now */ 1823c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1824c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return nsegs; 1825c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1826c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1827c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1828c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Prepost any receive buffer, then post send. 1829c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 1830c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Receive buffer is donated to hardware, reclaimed upon recv completion. 1831c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1832c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1833c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_post(struct rpcrdma_ia *ia, 1834c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ep *ep, 1835c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_req *req) 1836c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1837c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_send_wr send_wr, *send_wr_fail; 1838c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep *rep = req->rl_reply; 1839c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 1840c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1841c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rep) { 1842c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_ep_post_recv(ia, ep, rep); 1843c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1844c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1845c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply = NULL; 1846c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1847c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1848c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.next = NULL; 1849c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.wr_id = 0ULL; /* no send cookie */ 1850c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.sg_list = req->rl_send_iov; 1851c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.num_sge = req->rl_niovs; 1852c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.opcode = IB_WR_SEND; 1853c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (send_wr.num_sge == 4) /* no need to sync any pad (constant) */ 1854c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_sync_single_for_device(ia->ri_id->device, 1855c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_send_iov[3].addr, req->rl_send_iov[3].length, 1856c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ DMA_TO_DEVICE); 1857c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_sync_single_for_device(ia->ri_id->device, 1858c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_send_iov[1].addr, req->rl_send_iov[1].length, 1859c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ DMA_TO_DEVICE); 1860c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_sync_single_for_device(ia->ri_id->device, 1861c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_send_iov[0].addr, req->rl_send_iov[0].length, 1862c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ DMA_TO_DEVICE); 1863c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1864c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (DECR_CQCOUNT(ep) > 0) 1865c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.send_flags = 0; 1866c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ else { /* Provider must take a send completion every now and then */ 1867c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ INIT_CQCOUNT(ep); 1868c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.send_flags = IB_SEND_SIGNALED; 1869c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1870c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1871c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_post_send(ia->ri_id->qp, &send_wr, &send_wr_fail); 1872c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1873c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_post_send returned %i\n", __func__, 1874c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc); 1875c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out: 1876c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 1877c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1878c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1879c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1880c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * (Re)post a receive buffer. 1881c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1882c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1883c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_post_recv(struct rpcrdma_ia *ia, 1884c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ep *ep, 1885c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep *rep) 1886c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1887c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_recv_wr recv_wr, *recv_wr_fail; 1888c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 1889c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1890c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ recv_wr.next = NULL; 1891c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ recv_wr.wr_id = (u64) (unsigned long) rep; 1892c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ recv_wr.sg_list = &rep->rr_iov; 1893c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ recv_wr.num_sge = 1; 1894c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1895c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_sync_single_for_cpu(ia->ri_id->device, 1896c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_iov.addr, rep->rr_iov.length, DMA_BIDIRECTIONAL); 1897c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1898c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ DECR_CQCOUNT(ep); 1899c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_post_recv(ia->ri_id->qp, &recv_wr, &recv_wr_fail); 1900c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1901c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1902c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_post_recv returned %i\n", __func__, 1903c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc); 1904c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 1905c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1906