verbs.c revision bd7ed1d13304d914648dacec4dbb9145aaae614e
1f58851e6b0f148fb4b2a1c6f70beb2f125863c0f\"Talpey, Thomas\/* 2c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved. 3c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 4c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * This software is available to you under a choice of one of two 5c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * licenses. You may choose to be licensed under the terms of the GNU 6c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * General Public License (GPL) Version 2, available from the file 7c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * COPYING in the main directory of this source tree, or the BSD-type 8c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * license below: 9c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 10c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Redistribution and use in source and binary forms, with or without 11c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * modification, are permitted provided that the following conditions 12c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * are met: 13c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 14c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Redistributions of source code must retain the above copyright 15c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * notice, this list of conditions and the following disclaimer. 16c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 17c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Redistributions in binary form must reproduce the above 18c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * copyright notice, this list of conditions and the following 19c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * disclaimer in the documentation and/or other materials provided 20c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * with the distribution. 21c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 22c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Neither the name of the Network Appliance, Inc. nor the names of 23c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * its contributors may be used to endorse or promote products 24c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * derived from this software without specific prior written 25c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * permission. 26c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 27c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 28c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 29c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 30c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 31c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 32c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 33c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 34c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 35c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 36c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 37c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 38f58851e6b0f148fb4b2a1c6f70beb2f125863c0f\"Talpey, Thomas\ */ 39f58851e6b0f148fb4b2a1c6f70beb2f125863c0f\"Talpey, Thomas\ 40c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 41c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * verbs.c 42c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 43c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Encapsulates the major functions managing: 44c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o adapters 45c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o endpoints 46c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o connections 47c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o buffer memory 48c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 49c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 50c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#include <linux/pci.h> /* for Tavor hack below */ 51c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 52f58851e6b0f148fb4b2a1c6f70beb2f125863c0f\"Talpey, Thomas\#include "xprt_rdma.h" 53f58851e6b0f148fb4b2a1c6f70beb2f125863c0f\"Talpey, Thomas\ 54c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 55c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Globals/Macros 56c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 57c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 58c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#ifdef RPC_DEBUG 59c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\# define RPCDBG_FACILITY RPCDBG_TRANS 60c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#endif 61c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 62c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 63c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * internal functions 64c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 65c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 66c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 67c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * handle replies in tasklet context, using a single, global list 68c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * rdma tasklet function -- just turn around and call the func 69c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * for all replies on the list 70c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 71c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 72c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static DEFINE_SPINLOCK(rpcrdma_tk_lock_g); 73c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static LIST_HEAD(rpcrdma_tasklets_g); 74c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 75c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 76c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_run_tasklet(unsigned long data) 77c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 78c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep *rep; 79c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ void (*func)(struct rpcrdma_rep *); 80c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 81c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 82c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ data = data; 83c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&rpcrdma_tk_lock_g, flags); 84c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ while (!list_empty(&rpcrdma_tasklets_g)) { 85c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep = list_entry(rpcrdma_tasklets_g.next, 86c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep, rr_list); 87c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_del(&rep->rr_list); 88c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ func = rep->rr_func; 89c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_func = NULL; 90c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&rpcrdma_tk_lock_g, flags); 91c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 92c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (func) 93c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ func(rep); 94c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ else 95c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_recv_buffer_put(rep); 96c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 97c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&rpcrdma_tk_lock_g, flags); 98c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 99c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&rpcrdma_tk_lock_g, flags); 100c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 101c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 102c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static DECLARE_TASKLET(rpcrdma_tasklet_g, rpcrdma_run_tasklet, 0UL); 103c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 104c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static inline void 105c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_schedule_tasklet(struct rpcrdma_rep *rep) 106c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 107c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 108c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 109c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&rpcrdma_tk_lock_g, flags); 110c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_add_tail(&rep->rr_list, &rpcrdma_tasklets_g); 111c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&rpcrdma_tk_lock_g, flags); 112c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ tasklet_schedule(&rpcrdma_tasklet_g); 113c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 114c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 115c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 116c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_qp_async_error_upcall(struct ib_event *event, void *context) 117c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 118c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ep *ep = context; 119c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 120c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: QP error %X on device %s ep %p\n", 121c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, event->event, event->device->name, context); 122c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_connected == 1) { 123c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = -EIO; 124c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_func(ep); 125c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wake_up_all(&ep->rep_connect_wait); 126c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 127c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 128c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 129c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 130c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_cq_async_error_upcall(struct ib_event *event, void *context) 131c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 132c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ep *ep = context; 133c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 134c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: CQ error %X on device %s ep %p\n", 135c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, event->event, event->device->name, context); 136c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_connected == 1) { 137c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = -EIO; 138c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_func(ep); 139c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wake_up_all(&ep->rep_connect_wait); 140c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 141c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 142c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 143c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static inline 144c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void rpcrdma_event_process(struct ib_wc *wc) 145c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 146c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep *rep = 147c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (struct rpcrdma_rep *)(unsigned long) wc->wr_id; 148c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 149c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: event rep %p status %X opcode %X length %u\n", 150c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rep, wc->status, wc->opcode, wc->byte_len); 151c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 152c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (!rep) /* send or bind completion that we don't care about */ 153c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return; 154c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 155c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IB_WC_SUCCESS != wc->status) { 156c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: %s WC status %X, connection lost\n", 157c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, (wc->opcode & IB_WC_RECV) ? "recv" : "send", 158c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wc->status); 159c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_len = ~0U; 160c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_schedule_tasklet(rep); 161c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return; 162c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 163c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 164c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (wc->opcode) { 165c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case IB_WC_RECV: 166c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_len = wc->byte_len; 167c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_sync_single_for_cpu( 168c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rdmab_to_ia(rep->rr_buffer)->ri_id->device, 169c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_iov.addr, rep->rr_len, DMA_FROM_DEVICE); 170c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Keep (only) the most recent credits, after check validity */ 171c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rep->rr_len >= 16) { 172c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_msg *p = 173c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (struct rpcrdma_msg *) rep->rr_base; 174c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned int credits = ntohl(p->rm_credit); 175c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (credits == 0) { 176c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: server" 177c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " dropped credits to 0!\n", __func__); 178c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* don't deadlock */ 179c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ credits = 1; 180c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } else if (credits > rep->rr_buffer->rb_max_requests) { 181c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: server" 182c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " over-crediting: %d (%d)\n", 183c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, credits, 184c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_buffer->rb_max_requests); 185c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ credits = rep->rr_buffer->rb_max_requests; 186c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 187c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ atomic_set(&rep->rr_buffer->rb_credits, credits); 188c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 189c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* fall through */ 190c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case IB_WC_BIND_MW: 191c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_schedule_tasklet(rep); 192c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 193c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 194c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: unexpected WC event %X\n", 195c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, wc->opcode); 196c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 197c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 198c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 199c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 200c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static inline int 201c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_cq_poll(struct ib_cq *cq) 202c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 203c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_wc wc; 204c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 205c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 206c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ for (;;) { 207c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_poll_cq(cq, 1, &wc); 208c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc < 0) { 209c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_poll_cq failed %i\n", 210c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 211c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 212c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 213c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc == 0) 214c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 215c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 216c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_event_process(&wc); 217c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 218c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 219c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 220c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 221c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 222c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 223c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * rpcrdma_cq_event_upcall 224c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 225c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * This upcall handles recv, send, bind and unbind events. 226c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * It is reentrant but processes single events in order to maintain 227c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * ordering of receives to keep server credits. 228c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 229c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * It is the responsibility of the scheduled tasklet to return 230c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * recv buffers to the pool. NOTE: this affects synchronization of 231c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * connection shutdown. That is, the structures required for 232c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * the completion of the reply handler must remain intact until 233c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * all memory has been reclaimed. 234c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 235c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Note that send events are suppressed and do not result in an upcall. 236c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 237c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 238c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_cq_event_upcall(struct ib_cq *cq, void *context) 239c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 240c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 241c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 242c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_cq_poll(cq); 243c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 244c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return; 245c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 246c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_req_notify_cq(cq, IB_CQ_NEXT_COMP); 247c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 248c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_req_notify_cq failed %i\n", 249c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 250c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return; 251c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 252c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 253c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_cq_poll(cq); 254c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 255c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 256c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#ifdef RPC_DEBUG 257c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static const char * const conn[] = { 258c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "address resolved", 259c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "address error", 260c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "route resolved", 261c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "route error", 262c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "connect request", 263c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "connect response", 264c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "connect error", 265c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "unreachable", 266c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "rejected", 267c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "established", 268c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "disconnected", 269c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "device removal" 270c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\}; 271c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#endif 272c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 273c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static int 274c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_conn_upcall(struct rdma_cm_id *id, struct rdma_cm_event *event) 275c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 276c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_xprt *xprt = id->context; 277c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = &xprt->rx_ia; 278c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ep *ep = &xprt->rx_ep; 279c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct sockaddr_in *addr = (struct sockaddr_in *) &ep->rep_remote_addr; 280c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_qp_attr attr; 281c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_qp_init_attr iattr; 282c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int connstate = 0; 283c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 284c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (event->event) { 285c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_ADDR_RESOLVED: 286c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_ROUTE_RESOLVED: 287c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ complete(&ia->ri_done); 288c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 289c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_ADDR_ERROR: 290c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_async_rc = -EHOSTUNREACH; 291c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: CM address resolution error, ep 0x%p\n", 292c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, ep); 293c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ complete(&ia->ri_done); 294c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 295c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_ROUTE_ERROR: 296c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_async_rc = -ENETUNREACH; 297c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: CM route resolution error, ep 0x%p\n", 298c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, ep); 299c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ complete(&ia->ri_done); 300c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 301c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_ESTABLISHED: 302c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = 1; 303c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_query_qp(ia->ri_id->qp, &attr, 304c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ IB_QP_MAX_QP_RD_ATOMIC | IB_QP_MAX_DEST_RD_ATOMIC, 305c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &iattr); 306c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: %d responder resources" 307c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " (%d initiator)\n", 308c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, attr.max_dest_rd_atomic, attr.max_rd_atomic); 309c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto connected; 310c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_CONNECT_ERROR: 311c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = -ENOTCONN; 312c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto connected; 313c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_UNREACHABLE: 314c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = -ENETDOWN; 315c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto connected; 316c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_REJECTED: 317c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = -ECONNREFUSED; 318c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto connected; 319c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_DISCONNECTED: 320c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = -ECONNABORTED; 321c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto connected; 322c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RDMA_CM_EVENT_DEVICE_REMOVAL: 323c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ connstate = -ENODEV; 324c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\connected: 325c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: %s: %u.%u.%u.%u:%u" 326c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " (ep 0x%p event 0x%x)\n", 327c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, 328c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (event->event <= 11) ? conn[event->event] : 329c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "unknown connection error", 330c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ NIPQUAD(addr->sin_addr.s_addr), 331c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ntohs(addr->sin_port), 332c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep, event->event); 333c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ atomic_set(&rpcx_to_rdmax(ep->rep_xprt)->rx_buf.rb_credits, 1); 334c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: %sconnected\n", 335c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, connstate > 0 ? "" : "dis"); 336c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = connstate; 337c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_func(ep); 338c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wake_up_all(&ep->rep_connect_wait); 339c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 340c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 341c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_async_rc = -EINVAL; 342c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: unexpected CM event %X\n", 343c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, event->event); 344c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ complete(&ia->ri_done); 345c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 346c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 347c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 348c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 349c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 350c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 351c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static struct rdma_cm_id * 352c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_create_id(struct rpcrdma_xprt *xprt, 353c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia, struct sockaddr *addr) 354c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 355c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rdma_cm_id *id; 356c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 357c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 358c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ id = rdma_create_id(rpcrdma_conn_upcall, xprt, RDMA_PS_TCP); 359c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(id)) { 360c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(id); 361c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_create_id() failed %i\n", 362c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 363c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return id; 364c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 365c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 366c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_async_rc = 0; 367c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rdma_resolve_addr(id, NULL, addr, RDMA_RESOLVE_TIMEOUT); 368c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 369c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_resolve_addr() failed %i\n", 370c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 371c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 372c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 373c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wait_for_completion(&ia->ri_done); 374c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ia->ri_async_rc; 375c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 376c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 377c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 378c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_async_rc = 0; 379c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rdma_resolve_route(id, RDMA_RESOLVE_TIMEOUT); 380c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 381c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_resolve_route() failed %i\n", 382c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 383c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 384c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 385c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wait_for_completion(&ia->ri_done); 386c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ia->ri_async_rc; 387c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 388c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 389c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 390c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return id; 391c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 392c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out: 393c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rdma_destroy_id(id); 394c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return ERR_PTR(rc); 395c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 396c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 397c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 398c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Drain any cq, prior to teardown. 399c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 400c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 401c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_clean_cq(struct ib_cq *cq) 402c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 403c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_wc wc; 404c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int count = 0; 405c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 406c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ while (1 == ib_poll_cq(cq, 1, &wc)) 407c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ++count; 408c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 409c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (count) 410c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: flushed %d events (last 0x%x)\n", 411c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, count, wc.opcode); 412c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 413c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 414c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 415c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Exported functions. 416c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 417c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 418c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 419c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Open and initialize an Interface Adapter. 420c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o initializes fields of struct rpcrdma_ia, including 421c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * interface and provider attributes and protection zone. 422c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 423c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 424c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ia_open(struct rpcrdma_xprt *xprt, struct sockaddr *addr, int memreg) 425c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 426bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey int rc, mem_priv; 427bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey struct ib_device_attr devattr; 428c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = &xprt->rx_ia; 429c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 430c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ init_completion(&ia->ri_done); 431c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 432c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_id = rpcrdma_create_id(xprt, ia, addr); 433c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(ia->ri_id)) { 434c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(ia->ri_id); 435c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out1; 436c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 437c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 438c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_pd = ib_alloc_pd(ia->ri_id->device); 439c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(ia->ri_pd)) { 440c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(ia->ri_pd); 441c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_alloc_pd() failed %i\n", 442c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 443c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out2; 444c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 445c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 446c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 447bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey * Query the device to determine if the requested memory 448bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey * registration strategy is supported. If it isn't, set the 449bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey * strategy to a globally supported model. 450bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey */ 451bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey rc = ib_query_device(ia->ri_id->device, &devattr); 452bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey if (rc) { 453bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey dprintk("RPC: %s: ib_query_device failed %d\n", 454bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey __func__, rc); 455bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey goto out2; 456bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey } 457bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey 458bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey if (devattr.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY) { 459bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey ia->ri_have_dma_lkey = 1; 460bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey ia->ri_dma_lkey = ia->ri_id->device->local_dma_lkey; 461bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey } 462bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey 463bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey switch (memreg) { 464bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_MEMWINDOWS: 465bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_MEMWINDOWS_ASYNC: 466bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey if (!(devattr.device_cap_flags & IB_DEVICE_MEM_WINDOW)) { 467bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey dprintk("RPC: %s: MEMWINDOWS registration " 468bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey "specified but not supported by adapter, " 469bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey "using slower RPCRDMA_REGISTER\n", 470bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey __func__); 471bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey memreg = RPCRDMA_REGISTER; 472bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey } 473bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey break; 474bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_MTHCAFMR: 475bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey if (!ia->ri_id->device->alloc_fmr) { 476bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey#if RPCRDMA_PERSISTENT_REGISTRATION 477bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey dprintk("RPC: %s: MTHCAFMR registration " 478bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey "specified but not supported by adapter, " 479bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey "using riskier RPCRDMA_ALLPHYSICAL\n", 480bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey __func__); 481bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey memreg = RPCRDMA_ALLPHYSICAL; 482bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey#else 483bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey dprintk("RPC: %s: MTHCAFMR registration " 484bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey "specified but not supported by adapter, " 485bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey "using slower RPCRDMA_REGISTER\n", 486bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey __func__); 487bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey memreg = RPCRDMA_REGISTER; 488bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey#endif 489bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey } 490bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey break; 491bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey } 492bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey 493bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey /* 494c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Optionally obtain an underlying physical identity mapping in 495c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * order to do a memory window-based bind. This base registration 496c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * is protected from remote access - that is enabled only by binding 497c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * for the specific bytes targeted during each RPC operation, and 498c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * revoked after the corresponding completion similar to a storage 499c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * adapter. 500c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 501bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey switch (memreg) { 502bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_BOUNCEBUFFERS: 503bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_REGISTER: 504bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey break; 505c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#if RPCRDMA_PERSISTENT_REGISTRATION 506bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_ALLPHYSICAL: 507bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey mem_priv = IB_ACCESS_LOCAL_WRITE | 508bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey IB_ACCESS_REMOTE_WRITE | 509bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey IB_ACCESS_REMOTE_READ; 510bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey goto register_setup; 511c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#endif 512bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_MEMWINDOWS_ASYNC: 513bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_MEMWINDOWS: 514bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey mem_priv = IB_ACCESS_LOCAL_WRITE | 515bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey IB_ACCESS_MW_BIND; 516bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey goto register_setup; 517bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey case RPCRDMA_MTHCAFMR: 518bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey if (ia->ri_have_dma_lkey) 519c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 520bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey mem_priv = IB_ACCESS_LOCAL_WRITE; 521bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey register_setup: 522c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_bind_mem = ib_get_dma_mr(ia->ri_pd, mem_priv); 523c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(ia->ri_bind_mem)) { 524c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ printk(KERN_ALERT "%s: ib_get_dma_mr for " 525c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "phys register failed with %lX\n\t" 526c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "Will continue with degraded performance\n", 527c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, PTR_ERR(ia->ri_bind_mem)); 528c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ memreg = RPCRDMA_REGISTER; 529c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_bind_mem = NULL; 530c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 531bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey break; 532bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey default: 533bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey printk(KERN_ERR "%s: invalid memory registration mode %d\n", 534bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey __func__, memreg); 535bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey rc = -EINVAL; 536bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey goto out2; 537c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 538bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey dprintk("RPC: %s: memory registration strategy is %d\n", 539bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey __func__, memreg); 540c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 541c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Else will do memory reg/dereg for each chunk */ 542c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_memreg_strategy = memreg; 543c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 544c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 545c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out2: 546c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rdma_destroy_id(ia->ri_id); 547c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out1: 548c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 549c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 550c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 551c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 552c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Clean up/close an IA. 553c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o if event handles and PD have been initialized, free them. 554c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * o close the IA 555c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 556c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void 557c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ia_close(struct rpcrdma_ia *ia) 558c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 559c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 560c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 561c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: entering\n", __func__); 562c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ia->ri_bind_mem != NULL) { 563c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_dereg_mr(ia->ri_bind_mem); 564c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_dereg_mr returned %i\n", 565c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 566c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 567c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ia->ri_id != NULL && !IS_ERR(ia->ri_id) && ia->ri_id->qp) 568c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rdma_destroy_qp(ia->ri_id); 569c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ia->ri_pd != NULL && !IS_ERR(ia->ri_pd)) { 570c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_dealloc_pd(ia->ri_pd); 571c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_dealloc_pd returned %i\n", 572c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 573c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 574c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ia->ri_id != NULL && !IS_ERR(ia->ri_id)) 575c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rdma_destroy_id(ia->ri_id); 576c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 577c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 578c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 579c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Create unconnected endpoint. 580c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 581c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 582c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia, 583c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_create_data_internal *cdata) 584c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 585c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_device_attr devattr; 5865d40a8a525c8165bafed233cf0f137e8d10d7e92Chuck Lever int rc, err; 587c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 588c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_query_device(ia->ri_id->device, &devattr); 589c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 590c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_query_device failed %d\n", 591c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 592c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 593c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 594c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 595c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* check provider's send/recv wr limits */ 596c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (cdata->max_requests > devattr.max_qp_wr) 597c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ cdata->max_requests = devattr.max_qp_wr; 598c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 599c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.event_handler = rpcrdma_qp_async_error_upcall; 600c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.qp_context = ep; 601c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* send_cq and recv_cq initialized below */ 602c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.srq = NULL; 603c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_wr = cdata->max_requests; 604c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 605c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS_ASYNC: 606c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS: 607c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Add room for mw_binds+unbinds - overkill! */ 608c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_wr++; 609c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_wr *= (2 * RPCRDMA_MAX_SEGS); 610c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_attr.cap.max_send_wr > devattr.max_qp_wr) 611c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return -EINVAL; 612c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 613c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 614c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 615c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 616c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_recv_wr = cdata->max_requests; 617c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_sge = (cdata->padding ? 4 : 2); 618c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_recv_sge = 1; 619c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_inline_data = 0; 620c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.sq_sig_type = IB_SIGNAL_REQ_WR; 621c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.qp_type = IB_QPT_RC; 622c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.port_num = ~0; 623c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 624c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: requested max: dtos: send %d recv %d; " 625c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "iovs: send %d recv %d\n", 626c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, 627c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_wr, 628c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_recv_wr, 629c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_sge, 630c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_recv_sge); 631c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 632c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* set trigger for requesting send completion */ 633c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_cqinit = ep->rep_attr.cap.max_send_wr/2 /* - 1*/; 634c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 635c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS_ASYNC: 636c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS: 637c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_cqinit -= RPCRDMA_MAX_SEGS; 638c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 639c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 640c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 641c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 642c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_cqinit <= 2) 643c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_cqinit = 0; 644c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ INIT_CQCOUNT(ep); 645c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_ia = ia; 646c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ init_waitqueue_head(&ep->rep_connect_wait); 647c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 648c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 649c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Create a single cq for receive dto and mw_bind (only ever 650c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * care about unbind, really). Send completions are suppressed. 651c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Use single threaded tasklet upcalls to maintain ordering. 652c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 653c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_cq = ib_create_cq(ia->ri_id->device, rpcrdma_cq_event_upcall, 654c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_cq_async_error_upcall, NULL, 655c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_recv_wr + 656c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.cap.max_send_wr + 1, 0); 657c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(ep->rep_cq)) { 658c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(ep->rep_cq); 659c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_create_cq failed: %i\n", 660c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 661c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out1; 662c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 663c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 664c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_req_notify_cq(ep->rep_cq, IB_CQ_NEXT_COMP); 665c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 666c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_req_notify_cq failed: %i\n", 667c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 668c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out2; 669c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 670c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 671c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.send_cq = ep->rep_cq; 672c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_attr.recv_cq = ep->rep_cq; 673c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 674c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Initialize cma parameters */ 675c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 676c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* RPC/RDMA does not use private data */ 677c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.private_data = NULL; 678c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.private_data_len = 0; 679c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 680c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Client offers RDMA Read but does not initiate */ 681c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 682c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_BOUNCEBUFFERS: 683c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.responder_resources = 0; 684c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 685c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 686c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_REGISTER: 687c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.responder_resources = cdata->max_requests * 688c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (RPCRDMA_MAX_DATA_SEGS / 8); 689c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 690c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS: 691c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS_ASYNC: 692c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#if RPCRDMA_PERSISTENT_REGISTRATION 693c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_ALLPHYSICAL: 694c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#endif 695c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.responder_resources = cdata->max_requests * 696c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (RPCRDMA_MAX_DATA_SEGS / 2); 697c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 698c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 699c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 700c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 701c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_remote_cma.responder_resources > devattr.max_qp_rd_atom) 702c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.responder_resources = devattr.max_qp_rd_atom; 703c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.initiator_depth = 0; 704c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 705c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.retry_count = 7; 706c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.flow_control = 0; 707c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.rnr_retry_count = 0; 708c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 709c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 710c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 711c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out2: 7125d40a8a525c8165bafed233cf0f137e8d10d7e92Chuck Lever err = ib_destroy_cq(ep->rep_cq); 7135d40a8a525c8165bafed233cf0f137e8d10d7e92Chuck Lever if (err) 7145d40a8a525c8165bafed233cf0f137e8d10d7e92Chuck Lever dprintk("RPC: %s: ib_destroy_cq returned %i\n", 7155d40a8a525c8165bafed233cf0f137e8d10d7e92Chuck Lever __func__, err); 716c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out1: 717c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 718c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 719c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 720c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 721c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * rpcrdma_ep_destroy 722c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 723c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Disconnect and destroy endpoint. After this, the only 724c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * valid operations on the ep are to free it (if dynamically 725c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * allocated) or re-create it. 726c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 727c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * The caller's error handling must be sure to not leak the endpoint 728c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * if this function fails. 729c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 730c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 731c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_destroy(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) 732c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 733c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 734c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 735c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: entering, connected is %d\n", 736c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, ep->rep_connected); 737c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 738c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ia->ri_id->qp) { 739c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_ep_disconnect(ep, ia); 740c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 741c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rpcrdma_ep_disconnect" 742c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " returned %i\n", __func__, rc); 743c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 744c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 745c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_func = NULL; 746c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 747c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* padding - could be done in rpcrdma_buffer_destroy... */ 748c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_pad_mr) { 749c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_deregister_internal(ia, ep->rep_pad_mr, &ep->rep_pad); 750c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_pad_mr = NULL; 751c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 752c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 753c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ia->ri_id->qp) { 754c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rdma_destroy_qp(ia->ri_id); 755c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_id->qp = NULL; 756c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 757c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 758c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_clean_cq(ep->rep_cq); 759c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_destroy_cq(ep->rep_cq); 760c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 761c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_destroy_cq returned %i\n", 762c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 763c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 764c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 765c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 766c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 767c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 768c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Connect unconnected endpoint. 769c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 770c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 771c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) 772c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 773c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rdma_cm_id *id; 774c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc = 0; 775c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int retry_count = 0; 776c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int reconnect = (ep->rep_connected != 0); 777c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 778c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (reconnect) { 779c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_xprt *xprt; 780c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\retry: 781c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_ep_disconnect(ep, ia); 782c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc && rc != -ENOTCONN) 783c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rpcrdma_ep_disconnect" 784c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " status %i\n", __func__, rc); 785c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_clean_cq(ep->rep_cq); 786c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 787c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ xprt = container_of(ia, struct rpcrdma_xprt, rx_ia); 788c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ id = rpcrdma_create_id(xprt, ia, 789c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (struct sockaddr *)&xprt->rx_data.addr); 790c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(id)) { 791c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(id); 792c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 793c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 794c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* TEMP TEMP TEMP - fail if new device: 795c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Deregister/remarshal *all* requests! 796c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Close and recreate adapter, pd, etc! 797c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Re-determine all attributes still sane! 798c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * More stuff I haven't thought of! 799c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Rrrgh! 800c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 801c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ia->ri_id->device != id->device) { 802c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ printk("RPC: %s: can't reconnect on " 803c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "different device!\n", __func__); 804c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rdma_destroy_id(id); 805c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = -ENETDOWN; 806c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 807c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 808c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* END TEMP */ 809c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rdma_destroy_id(ia->ri_id); 810c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ia->ri_id = id; 811c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 812c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 813c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rdma_create_qp(ia->ri_id, ia->ri_pd, &ep->rep_attr); 814c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 815c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_create_qp failed %i\n", 816c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 817c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 818c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 819c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 820c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* XXX Tavor device performs badly with 2K MTU! */ 821c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\if (strnicmp(ia->ri_id->device->dma_device->bus->name, "pci", 3) == 0) { 822c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct pci_dev *pcid = to_pci_dev(ia->ri_id->device->dma_device); 823c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (pcid->device == PCI_DEVICE_ID_MELLANOX_TAVOR && 824c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (pcid->vendor == PCI_VENDOR_ID_MELLANOX || 825c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ pcid->vendor == PCI_VENDOR_ID_TOPSPIN)) { 826c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_qp_attr attr = { 827c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ .path_mtu = IB_MTU_1024 828c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ }; 829c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_modify_qp(ia->ri_id->qp, &attr, IB_QP_PATH_MTU); 830c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 831c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 832c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 833c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Theoretically a client initiator_depth > 0 is not needed, 834c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * but many peers fail to complete the connection unless they 835c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * == responder_resources! */ 836c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_remote_cma.initiator_depth != 837c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.responder_resources) 838c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.initiator_depth = 839c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.responder_resources; 840c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 841c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = 0; 842c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 843c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rdma_connect(ia->ri_id, &ep->rep_remote_cma); 844c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) { 845c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_connect() failed with %i\n", 846c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 847c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 848c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 849c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 850c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (reconnect) 851c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 852c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 853c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wait_event_interruptible(ep->rep_connect_wait, ep->rep_connected != 0); 854c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 855c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 856c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Check state. A non-peer reject indicates no listener 857c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * (ECONNREFUSED), which may be a transient state. All 858c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * others indicate a transport condition which has already 859c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * undergone a best-effort. 860c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 861c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_connected == -ECONNREFUSED 862c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ && ++retry_count <= RDMA_CONNECT_RETRY_MAX) { 863c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: non-peer_reject, retry\n", __func__); 864c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto retry; 865c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 866c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_connected <= 0) { 867c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Sometimes, the only way to reliably connect to remote 868c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * CMs is to use same nonzero values for ORD and IRD. */ 869c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.initiator_depth = 870c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_remote_cma.responder_resources; 871c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_remote_cma.initiator_depth == 0) 872c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ++ep->rep_remote_cma.initiator_depth; 873c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (ep->rep_remote_cma.responder_resources == 0) 874c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ++ep->rep_remote_cma.responder_resources; 875c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (retry_count++ == 0) 876c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto retry; 877c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ep->rep_connected; 878c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } else { 879c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: connected\n", __func__); 880c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 881c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 882c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out: 883c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 884c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = rc; 885c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 886c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 887c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 888c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 889c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * rpcrdma_ep_disconnect 890c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 891c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * This is separate from destroy to facilitate the ability 892c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * to reconnect without recreating the endpoint. 893c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 894c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * This call is not reentrant, and must not be made in parallel 895c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * on the same endpoint. 896c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 897c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 898c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_disconnect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) 899c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 900c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 901c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 902c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_clean_cq(ep->rep_cq); 903c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rdma_disconnect(ia->ri_id); 904c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (!rc) { 905c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* returns without wait if not connected */ 906c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ wait_event_interruptible(ep->rep_connect_wait, 907c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected != 1); 908c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: after wait, %sconnected\n", __func__, 909c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (ep->rep_connected == 1) ? "still " : "dis"); 910c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } else { 911c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: rdma_disconnect %i\n", __func__, rc); 912c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ep->rep_connected = rc; 913c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 914c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 915c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 916c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 917c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 918c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Initialize buffer memory 919c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 920c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 921c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_buffer_create(struct rpcrdma_buffer *buf, struct rpcrdma_ep *ep, 922c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia, struct rpcrdma_create_data_internal *cdata) 923c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 924c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ char *p; 925c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ size_t len; 926c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int i, rc; 9278d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_mw *r; 928c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 929c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_max_requests = cdata->max_requests; 930c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_init(&buf->rb_lock); 931c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ atomic_set(&buf->rb_credits, 1); 932c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 933c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Need to allocate: 934c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 1. arrays for send and recv pointers 935c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 2. arrays of struct rpcrdma_req to fill in pointers 936c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 3. array of struct rpcrdma_rep for replies 937c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 4. padding, if any 9388d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey * 5. mw's or fmr's, if any 939c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Send/recv buffers in req/rep need to be registered 940c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 941c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 942c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len = buf->rb_max_requests * 943c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ (sizeof(struct rpcrdma_req *) + sizeof(struct rpcrdma_rep *)); 944c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len += cdata->padding; 945c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 946c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 947c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* TBD we are perhaps overallocating here */ 948c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len += (buf->rb_max_requests + 1) * RPCRDMA_MAX_SEGS * 949c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ sizeof(struct rpcrdma_mw); 950c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 951c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS_ASYNC: 952c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS: 953c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len += (buf->rb_max_requests + 1) * RPCRDMA_MAX_SEGS * 954c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ sizeof(struct rpcrdma_mw); 955c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 956c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 957c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 958c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 959c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 960c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* allocate 1, 4 and 5 in one shot */ 961c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ p = kzalloc(len, GFP_KERNEL); 962c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (p == NULL) { 963c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: req_t/rep_t/pad kzalloc(%zd) failed\n", 964c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, len); 965c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = -ENOMEM; 966c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 967c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 968c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_pool = p; /* for freeing it later */ 969c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 970c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_send_bufs = (struct rpcrdma_req **) p; 971c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ p = (char *) &buf->rb_send_bufs[buf->rb_max_requests]; 972c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_recv_bufs = (struct rpcrdma_rep **) p; 973c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ p = (char *) &buf->rb_recv_bufs[buf->rb_max_requests]; 974c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 975c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 976c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Register the zeroed pad buffer, if any. 977c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 978c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (cdata->padding) { 979c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_register_internal(ia, p, cdata->padding, 980c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &ep->rep_pad_mr, &ep->rep_pad); 981c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 982c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 983c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 984c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ p += cdata->padding; 985c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 986c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 987c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Allocate the fmr's, or mw's for mw_bind chunk registration. 988c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * We "cycle" the mw's in order to minimize rkey reuse, 989c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * and also reduce unbind-to-bind collision. 990c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 991c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ INIT_LIST_HEAD(&buf->rb_mws); 9928d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey r = (struct rpcrdma_mw *)p; 993c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 994c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 995c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* TBD we are perhaps overallocating here */ 996c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ for (i = (buf->rb_max_requests+1) * RPCRDMA_MAX_SEGS; i; i--) { 9978d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey static struct ib_fmr_attr fa = 9988d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey { RPCRDMA_MAX_DATA_SEGS, 1, PAGE_SHIFT }; 999c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ r->r.fmr = ib_alloc_fmr(ia->ri_pd, 1000c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ, 1001c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &fa); 1002c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(r->r.fmr)) { 1003c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(r->r.fmr); 1004c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_alloc_fmr" 1005c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " failed %i\n", __func__, rc); 1006c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1007c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1008c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_add(&r->mw_list, &buf->rb_mws); 1009c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ++r; 1010c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1011c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1012c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS_ASYNC: 1013c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS: 1014c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Allocate one extra request's worth, for full cycling */ 1015c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ for (i = (buf->rb_max_requests+1) * RPCRDMA_MAX_SEGS; i; i--) { 1016c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ r->r.mw = ib_alloc_mw(ia->ri_pd); 1017c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(r->r.mw)) { 1018c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(r->r.mw); 1019c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_alloc_mw" 1020c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " failed %i\n", __func__, rc); 1021c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1022c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1023c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_add(&r->mw_list, &buf->rb_mws); 1024c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ++r; 1025c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1026c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1027c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 1028c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1029c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1030c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1031c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 1032c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Allocate/init the request/reply buffers. Doing this 1033c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * using kmalloc for now -- one for each buf. 1034c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1035c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ for (i = 0; i < buf->rb_max_requests; i++) { 1036c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_req *req; 1037c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep *rep; 1038c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1039c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len = cdata->inline_wsize + sizeof(struct rpcrdma_req); 1040c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* RPC layer requests *double* size + 1K RPC_SLACK_SPACE! */ 1041c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Typical ~2400b, so rounding up saves work later */ 1042c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (len < 4096) 1043c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len = 4096; 1044c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req = kmalloc(len, GFP_KERNEL); 1045c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (req == NULL) { 1046c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: request buffer %d alloc" 1047c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " failed\n", __func__, i); 1048c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = -ENOMEM; 1049c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1050c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1051c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ memset(req, 0, sizeof(struct rpcrdma_req)); 1052c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_send_bufs[i] = req; 1053c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_send_bufs[i]->rl_buffer = buf; 1054c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1055c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_register_internal(ia, req->rl_base, 1056c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len - offsetof(struct rpcrdma_req, rl_base), 1057c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_send_bufs[i]->rl_handle, 1058c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_send_bufs[i]->rl_iov); 1059c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1060c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1061c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1062c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_send_bufs[i]->rl_size = len-sizeof(struct rpcrdma_req); 1063c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1064c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len = cdata->inline_rsize + sizeof(struct rpcrdma_rep); 1065c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep = kmalloc(len, GFP_KERNEL); 1066c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rep == NULL) { 1067c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: reply buffer %d alloc failed\n", 1068c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, i); 1069c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = -ENOMEM; 1070c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1071c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1072c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ memset(rep, 0, sizeof(struct rpcrdma_rep)); 1073c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_recv_bufs[i] = rep; 1074c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_recv_bufs[i]->rr_buffer = buf; 1075c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ init_waitqueue_head(&rep->rr_unbind); 1076c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1077c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_register_internal(ia, rep->rr_base, 1078c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ len - offsetof(struct rpcrdma_rep, rr_base), 1079c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_recv_bufs[i]->rr_handle, 1080c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_recv_bufs[i]->rr_iov); 1081c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1082c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1083c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1084c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1085c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: max_requests %d\n", 1086c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, buf->rb_max_requests); 1087c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* done */ 1088c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 1089c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out: 1090c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_buffer_destroy(buf); 1091c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 1092c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1093c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1094c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1095c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Unregister and destroy buffer memory. Need to deal with 1096c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * partial initialization, so it's callable from failed create. 1097c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Must be called before destroying endpoint, as registrations 1098c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * reference it. 1099c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1100c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void 1101c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf) 1102c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1103c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc, i; 1104c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = rdmab_to_ia(buf); 11058d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_mw *r; 1106c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1107c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* clean up in reverse order from create 1108c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 1. recv mr memory (mr free, then kfree) 1109c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 1a. bind mw memory 1110c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 2. send mr memory (mr free, then kfree) 1111c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 3. padding (if any) [moved to rpcrdma_ep_destroy] 1112c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 4. arrays 1113c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1114c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: entering\n", __func__); 1115c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1116c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ for (i = 0; i < buf->rb_max_requests; i++) { 1117c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (buf->rb_recv_bufs && buf->rb_recv_bufs[i]) { 1118c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_deregister_internal(ia, 1119c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_recv_bufs[i]->rr_handle, 1120c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_recv_bufs[i]->rr_iov); 1121c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ kfree(buf->rb_recv_bufs[i]); 1122c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1123c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (buf->rb_send_bufs && buf->rb_send_bufs[i]) { 1124c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ while (!list_empty(&buf->rb_mws)) { 1125c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ r = list_entry(buf->rb_mws.next, 1126c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_mw, mw_list); 1127c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_del(&r->mw_list); 1128c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 1129c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 1130c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_dealloc_fmr(r->r.fmr); 1131c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1132c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s:" 1133c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " ib_dealloc_fmr" 1134c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " failed %i\n", 1135c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 1136c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1137c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS_ASYNC: 1138c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS: 1139c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_dealloc_mw(r->r.mw); 1140c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1141c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s:" 1142c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " ib_dealloc_mw" 1143c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ " failed %i\n", 1144c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, rc); 1145c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1146c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 1147c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1148c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1149c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1150c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_deregister_internal(ia, 1151c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buf->rb_send_bufs[i]->rl_handle, 1152c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buf->rb_send_bufs[i]->rl_iov); 1153c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ kfree(buf->rb_send_bufs[i]); 1154c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1155c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1156c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1157c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ kfree(buf->rb_pool); 1158c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1159c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1160c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1161c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Get a set of request/reply buffers. 1162c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 1163c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Reply buffer (if needed) is attached to send buffer upon return. 1164c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Rule: 1165c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * rb_send_index and rb_recv_index MUST always be pointing to the 1166c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * *next* available buffer (non-NULL). They are incremented after 1167c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * removing buffers, and decremented *before* returning them. 1168c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1169c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\struct rpcrdma_req * 1170c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_buffer_get(struct rpcrdma_buffer *buffers) 1171c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1172c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_req *req; 1173c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 11748d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int i; 11758d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_mw *r; 1176c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1177c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&buffers->rb_lock, flags); 1178c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (buffers->rb_send_index == buffers->rb_max_requests) { 1179c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&buffers->rb_lock, flags); 1180c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: out of request buffers\n", __func__); 1181c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return ((struct rpcrdma_req *)NULL); 1182c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1183c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1184c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req = buffers->rb_send_bufs[buffers->rb_send_index]; 1185c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (buffers->rb_send_index < buffers->rb_recv_index) { 1186c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: %d extra receives outstanding (ok)\n", 1187c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ __func__, 1188c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_recv_index - buffers->rb_send_index); 1189c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply = NULL; 1190c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } else { 1191c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply = buffers->rb_recv_bufs[buffers->rb_recv_index]; 1192c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_recv_bufs[buffers->rb_recv_index++] = NULL; 1193c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1194c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_send_bufs[buffers->rb_send_index++] = NULL; 1195c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (!list_empty(&buffers->rb_mws)) { 11968d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey i = RPCRDMA_MAX_SEGS - 1; 1197c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ do { 1198c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ r = list_entry(buffers->rb_mws.next, 1199c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_mw, mw_list); 1200c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_del(&r->mw_list); 1201c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_segments[i].mr_chunk.rl_mw = r; 1202c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } while (--i >= 0); 1203c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1204c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&buffers->rb_lock, flags); 1205c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return req; 1206c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1207c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1208c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1209c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Put request/reply buffers back into pool. 1210c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Pre-decrement counter/array index. 1211c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1212c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void 1213c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_buffer_put(struct rpcrdma_req *req) 1214c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1215c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_buffer *buffers = req->rl_buffer; 1216c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = rdmab_to_ia(buffers); 1217c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int i; 1218c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 1219c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1220c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ BUG_ON(req->rl_nchunks != 0); 1221c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&buffers->rb_lock, flags); 1222c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_send_bufs[--buffers->rb_send_index] = req; 1223c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_niovs = 0; 1224c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (req->rl_reply) { 1225c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_recv_bufs[--buffers->rb_recv_index] = req->rl_reply; 1226c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ init_waitqueue_head(&req->rl_reply->rr_unbind); 1227c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply->rr_func = NULL; 1228c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply = NULL; 1229c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1230c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 1231c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 1232c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS_ASYNC: 1233c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS: 1234c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 1235c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Cycle mw's back in reverse order, and "spin" them. 1236c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * This delays and scrambles reuse as much as possible. 1237c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1238c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ i = 1; 1239c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ do { 1240c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_mw **mw; 1241c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ mw = &req->rl_segments[i].mr_chunk.rl_mw; 1242c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_add_tail(&(*mw)->mw_list, &buffers->rb_mws); 1243c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ *mw = NULL; 1244c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } while (++i < RPCRDMA_MAX_SEGS); 1245c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ list_add_tail(&req->rl_segments[0].mr_chunk.rl_mw->mw_list, 1246c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ &buffers->rb_mws); 1247c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_segments[0].mr_chunk.rl_mw = NULL; 1248c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1249c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 1250c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1251c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1252c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&buffers->rb_lock, flags); 1253c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1254c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1255c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1256c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Recover reply buffers from pool. 1257c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * This happens when recovering from error conditions. 1258c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Post-increment counter/array index. 1259c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1260c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void 1261c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_recv_buffer_get(struct rpcrdma_req *req) 1262c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1263c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_buffer *buffers = req->rl_buffer; 1264c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 1265c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1266c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (req->rl_iov.length == 0) /* special case xprt_rdma_allocate() */ 1267c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers = ((struct rpcrdma_req *) buffers)->rl_buffer; 1268c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&buffers->rb_lock, flags); 1269c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (buffers->rb_recv_index < buffers->rb_max_requests) { 1270c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply = buffers->rb_recv_bufs[buffers->rb_recv_index]; 1271c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_recv_bufs[buffers->rb_recv_index++] = NULL; 1272c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1273c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&buffers->rb_lock, flags); 1274c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1275c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1276c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1277c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Put reply buffers back into pool when not attached to 1278c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * request. This happens in error conditions, and when 1279c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * aborting unbinds. Pre-decrement counter/array index. 1280c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1281c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\void 1282c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep) 1283c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1284c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_buffer *buffers = rep->rr_buffer; 1285c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ unsigned long flags; 1286c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1287c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_func = NULL; 1288c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_lock_irqsave(&buffers->rb_lock, flags); 1289c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ buffers->rb_recv_bufs[--buffers->rb_recv_index] = rep; 1290c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ spin_unlock_irqrestore(&buffers->rb_lock, flags); 1291c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1292c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1293c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1294c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Wrappers for internal-use kmalloc memory registration, used by buffer code. 1295c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1296c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1297c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1298c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_register_internal(struct rpcrdma_ia *ia, void *va, int len, 1299c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_mr **mrp, struct ib_sge *iov) 1300c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1301c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_phys_buf ipb; 1302c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_mr *mr; 1303c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 1304c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1305c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* 1306c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * All memory passed here was kmalloc'ed, therefore phys-contiguous. 1307c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1308c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ iov->addr = ib_dma_map_single(ia->ri_id->device, 1309c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ va, len, DMA_BIDIRECTIONAL); 1310c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ iov->length = len; 1311c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1312bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey if (ia->ri_have_dma_lkey) { 1313bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey *mrp = NULL; 1314bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey iov->lkey = ia->ri_dma_lkey; 1315bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey return 0; 1316bd7ed1d13304d914648dacec4dbb9145aaae614eTom Talpey } else if (ia->ri_bind_mem != NULL) { 1317c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ *mrp = NULL; 1318c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ iov->lkey = ia->ri_bind_mem->lkey; 1319c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 1320c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1321c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1322c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ipb.addr = iov->addr; 1323c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ipb.size = iov->length; 1324c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ mr = ib_reg_phys_mr(ia->ri_pd, &ipb, 1, 1325c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ IB_ACCESS_LOCAL_WRITE, &iov->addr); 1326c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1327c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: phys convert: 0x%llx " 1328c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ "registered 0x%llx length %d\n", 1329a56daeb7d5c9a05b1cb52ae4bcca05fb6545656eAndrew Morton __func__, (unsigned long long)ipb.addr, 1330a56daeb7d5c9a05b1cb52ae4bcca05fb6545656eAndrew Morton (unsigned long long)iov->addr, len); 1331c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1332c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (IS_ERR(mr)) { 1333c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ *mrp = NULL; 1334c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = PTR_ERR(mr); 1335c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: failed with %i\n", __func__, rc); 1336c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } else { 1337c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ *mrp = mr; 1338c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ iov->lkey = mr->lkey; 1339c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = 0; 1340c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1341c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1342c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 1343c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1344c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1345c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1346c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_deregister_internal(struct rpcrdma_ia *ia, 1347c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_mr *mr, struct ib_sge *iov) 1348c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1349c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 1350c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1351c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_unmap_single(ia->ri_id->device, 1352c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ iov->addr, iov->length, DMA_BIDIRECTIONAL); 1353c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1354c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (NULL == mr) 1355c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return 0; 1356c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1357c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_dereg_mr(mr); 1358c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1359c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_dereg_mr failed %i\n", __func__, rc); 1360c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 1361c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1362c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1363c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1364c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Wrappers for chunk registration, shared by read/write chunk code. 1365c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1366c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1367c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 1368c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_map_one(struct rpcrdma_ia *ia, struct rpcrdma_mr_seg *seg, int writing) 1369c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1370c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dir = writing ? DMA_FROM_DEVICE : DMA_TO_DEVICE; 1371c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dmalen = seg->mr_len; 1372c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (seg->mr_page) 1373c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dma = ib_dma_map_page(ia->ri_id->device, 1374c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_page, offset_in_page(seg->mr_offset), 1375c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dmalen, seg->mr_dir); 1376c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ else 1377c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dma = ib_dma_map_single(ia->ri_id->device, 1378c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_offset, 1379c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dmalen, seg->mr_dir); 1380c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1381c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1382c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\static void 1383c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_unmap_one(struct rpcrdma_ia *ia, struct rpcrdma_mr_seg *seg) 1384c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1385c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (seg->mr_page) 1386c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_unmap_page(ia->ri_id->device, 1387c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dma, seg->mr_dmalen, seg->mr_dir); 1388c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ else 1389c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_unmap_single(ia->ri_id->device, 1390c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_dma, seg->mr_dmalen, seg->mr_dir); 1391c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1392c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 13938d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeystatic int 13948d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeyrpcrdma_register_fmr_external(struct rpcrdma_mr_seg *seg, 13958d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int *nsegs, int writing, struct rpcrdma_ia *ia) 13968d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey{ 13978d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_mr_seg *seg1 = seg; 13988d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey u64 physaddrs[RPCRDMA_MAX_DATA_SEGS]; 13998d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int len, pageoff, i, rc; 14008d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 14018d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey pageoff = offset_in_page(seg1->mr_offset); 14028d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_offset -= pageoff; /* start of page */ 14038d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_len += pageoff; 14048d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey len = -pageoff; 14058d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (*nsegs > RPCRDMA_MAX_DATA_SEGS) 14068d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey *nsegs = RPCRDMA_MAX_DATA_SEGS; 14078d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey for (i = 0; i < *nsegs;) { 14088d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_map_one(ia, seg, writing); 14098d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey physaddrs[i] = seg->mr_dma; 14108d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey len += seg->mr_len; 14118d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey ++seg; 14128d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey ++i; 14138d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey /* Check for holes */ 14148d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if ((i < *nsegs && offset_in_page(seg->mr_offset)) || 14158d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len)) 14168d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey break; 14178d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } 14188d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = ib_map_phys_fmr(seg1->mr_chunk.rl_mw->r.fmr, 14198d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey physaddrs, i, seg1->mr_dma); 14208d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (rc) { 14218d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey dprintk("RPC: %s: failed ib_map_phys_fmr " 14228d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey "%u@0x%llx+%i (%d)... status %i\n", __func__, 14238d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey len, (unsigned long long)seg1->mr_dma, 14248d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey pageoff, i, rc); 14258d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey while (i--) 14268d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_unmap_one(ia, --seg); 14278d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } else { 14288d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_rkey = seg1->mr_chunk.rl_mw->r.fmr->rkey; 14298d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_base = seg1->mr_dma + pageoff; 14308d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_nsegs = i; 14318d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_len = len; 14328d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } 14338d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey *nsegs = i; 14348d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey return rc; 14358d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey} 14368d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 14378d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeystatic int 14388d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeyrpcrdma_deregister_fmr_external(struct rpcrdma_mr_seg *seg, 14398d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_ia *ia) 14408d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey{ 14418d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_mr_seg *seg1 = seg; 14428d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey LIST_HEAD(l); 14438d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int rc; 14448d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 14458d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey list_add(&seg1->mr_chunk.rl_mw->r.fmr->list, &l); 14468d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = ib_unmap_fmr(&l); 14478d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey while (seg1->mr_nsegs--) 14488d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_unmap_one(ia, seg++); 14498d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (rc) 14508d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey dprintk("RPC: %s: failed ib_unmap_fmr," 14518d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey " status %i\n", __func__, rc); 14528d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey return rc; 14538d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey} 14548d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 14558d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeystatic int 14568d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeyrpcrdma_register_memwin_external(struct rpcrdma_mr_seg *seg, 14578d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int *nsegs, int writing, struct rpcrdma_ia *ia, 14588d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_xprt *r_xprt) 14598d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey{ 14608d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int mem_priv = (writing ? IB_ACCESS_REMOTE_WRITE : 14618d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey IB_ACCESS_REMOTE_READ); 14628d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct ib_mw_bind param; 14638d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int rc; 14648d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 14658d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey *nsegs = 1; 14668d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_map_one(ia, seg, writing); 14678d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.mr = ia->ri_bind_mem; 14688d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.wr_id = 0ULL; /* no send cookie */ 14698d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.addr = seg->mr_dma; 14708d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.length = seg->mr_len; 14718d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.send_flags = 0; 14728d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.mw_access_flags = mem_priv; 14738d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 14748d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey DECR_CQCOUNT(&r_xprt->rx_ep); 14758d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = ib_bind_mw(ia->ri_id->qp, seg->mr_chunk.rl_mw->r.mw, ¶m); 14768d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (rc) { 14778d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey dprintk("RPC: %s: failed ib_bind_mw " 14788d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey "%u@0x%llx status %i\n", 14798d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey __func__, seg->mr_len, 14808d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey (unsigned long long)seg->mr_dma, rc); 14818d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_unmap_one(ia, seg); 14828d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } else { 14838d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg->mr_rkey = seg->mr_chunk.rl_mw->r.mw->rkey; 14848d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg->mr_base = param.addr; 14858d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg->mr_nsegs = 1; 14868d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } 14878d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey return rc; 14888d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey} 14898d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 14908d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeystatic int 14918d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeyrpcrdma_deregister_memwin_external(struct rpcrdma_mr_seg *seg, 14928d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_ia *ia, 14938d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_xprt *r_xprt, void **r) 14948d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey{ 14958d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct ib_mw_bind param; 14968d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey LIST_HEAD(l); 14978d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int rc; 14988d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 14998d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey BUG_ON(seg->mr_nsegs != 1); 15008d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.mr = ia->ri_bind_mem; 15018d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.addr = 0ULL; /* unbind */ 15028d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.length = 0; 15038d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.mw_access_flags = 0; 15048d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (*r) { 15058d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.wr_id = (u64) (unsigned long) *r; 15068d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.send_flags = IB_SEND_SIGNALED; 15078d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey INIT_CQCOUNT(&r_xprt->rx_ep); 15088d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } else { 15098d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.wr_id = 0ULL; 15108d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey param.send_flags = 0; 15118d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey DECR_CQCOUNT(&r_xprt->rx_ep); 15128d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } 15138d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = ib_bind_mw(ia->ri_id->qp, seg->mr_chunk.rl_mw->r.mw, ¶m); 15148d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_unmap_one(ia, seg); 15158d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (rc) 15168d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey dprintk("RPC: %s: failed ib_(un)bind_mw," 15178d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey " status %i\n", __func__, rc); 15188d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey else 15198d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey *r = NULL; /* will upcall on completion */ 15208d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey return rc; 15218d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey} 15228d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 15238d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeystatic int 15248d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeyrpcrdma_register_default_external(struct rpcrdma_mr_seg *seg, 15258d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int *nsegs, int writing, struct rpcrdma_ia *ia) 15268d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey{ 15278d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int mem_priv = (writing ? IB_ACCESS_REMOTE_WRITE : 15288d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey IB_ACCESS_REMOTE_READ); 15298d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_mr_seg *seg1 = seg; 15308d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct ib_phys_buf ipb[RPCRDMA_MAX_DATA_SEGS]; 15318d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int len, i, rc = 0; 15328d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 15338d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (*nsegs > RPCRDMA_MAX_DATA_SEGS) 15348d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey *nsegs = RPCRDMA_MAX_DATA_SEGS; 15358d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey for (len = 0, i = 0; i < *nsegs;) { 15368d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_map_one(ia, seg, writing); 15378d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey ipb[i].addr = seg->mr_dma; 15388d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey ipb[i].size = seg->mr_len; 15398d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey len += seg->mr_len; 15408d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey ++seg; 15418d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey ++i; 15428d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey /* Check for holes */ 15438d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if ((i < *nsegs && offset_in_page(seg->mr_offset)) || 15448d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey offset_in_page((seg-1)->mr_offset+(seg-1)->mr_len)) 15458d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey break; 15468d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } 15478d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_base = seg1->mr_dma; 15488d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_chunk.rl_mr = ib_reg_phys_mr(ia->ri_pd, 15498d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey ipb, i, mem_priv, &seg1->mr_base); 15508d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (IS_ERR(seg1->mr_chunk.rl_mr)) { 15518d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = PTR_ERR(seg1->mr_chunk.rl_mr); 15528d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey dprintk("RPC: %s: failed ib_reg_phys_mr " 15538d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey "%u@0x%llx (%d)... status %i\n", 15548d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey __func__, len, 15558d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey (unsigned long long)seg1->mr_dma, i, rc); 15568d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey while (i--) 15578d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_unmap_one(ia, --seg); 15588d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } else { 15598d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_rkey = seg1->mr_chunk.rl_mr->rkey; 15608d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_nsegs = i; 15618d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_len = len; 15628d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey } 15638d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey *nsegs = i; 15648d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey return rc; 15658d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey} 15668d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 15678d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeystatic int 15688d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpeyrpcrdma_deregister_default_external(struct rpcrdma_mr_seg *seg, 15698d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_ia *ia) 15708d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey{ 15718d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey struct rpcrdma_mr_seg *seg1 = seg; 15728d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey int rc; 15738d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 15748d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = ib_dereg_mr(seg1->mr_chunk.rl_mr); 15758d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey seg1->mr_chunk.rl_mr = NULL; 15768d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey while (seg1->mr_nsegs--) 15778d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rpcrdma_unmap_one(ia, seg++); 15788d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey if (rc) 15798d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey dprintk("RPC: %s: failed ib_dereg_mr," 15808d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey " status %i\n", __func__, rc); 15818d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey return rc; 15828d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey} 15838d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey 1584c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1585c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_register_external(struct rpcrdma_mr_seg *seg, 1586c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int nsegs, int writing, struct rpcrdma_xprt *r_xprt) 1587c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1588c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = &r_xprt->rx_ia; 1589c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc = 0; 1590c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1591c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 1592c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1593c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#if RPCRDMA_PERSISTENT_REGISTRATION 1594c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_ALLPHYSICAL: 1595c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_map_one(ia, seg, writing); 1596c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_rkey = ia->ri_bind_mem->rkey; 1597c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_base = seg->mr_dma; 1598c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ seg->mr_nsegs = 1; 1599c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ nsegs = 1; 1600c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1601c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#endif 1602c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 16038d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey /* Registration using fmr memory registration */ 1604c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 16058d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = rpcrdma_register_fmr_external(seg, &nsegs, writing, ia); 1606c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1607c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1608c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Registration using memory windows */ 1609c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS_ASYNC: 1610c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS: 16118d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = rpcrdma_register_memwin_external(seg, &nsegs, writing, ia, r_xprt); 1612c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1613c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1614c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ /* Default registration each time */ 1615c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 16168d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = rpcrdma_register_default_external(seg, &nsegs, writing, ia); 1617c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1618c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1619c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1620c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return -1; 1621c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1622c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return nsegs; 1623c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1624c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1625c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1626c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_deregister_external(struct rpcrdma_mr_seg *seg, 1627c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_xprt *r_xprt, void *r) 1628c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1629c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ia *ia = &r_xprt->rx_ia; 1630c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int nsegs = seg->mr_nsegs, rc; 1631c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1632c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ switch (ia->ri_memreg_strategy) { 1633c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1634c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#if RPCRDMA_PERSISTENT_REGISTRATION 1635c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_ALLPHYSICAL: 1636c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ BUG_ON(nsegs != 1); 1637c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rpcrdma_unmap_one(ia, seg); 1638c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = 0; 1639c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1640c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\#endif 1641c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1642c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MTHCAFMR: 16438d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = rpcrdma_deregister_fmr_external(seg, ia); 1644c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1645c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1646c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS_ASYNC: 1647c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ case RPCRDMA_MEMWINDOWS: 16488d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = rpcrdma_deregister_memwin_external(seg, ia, r_xprt, &r); 1649c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1650c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1651c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ default: 16528d4ba0347ccfea4f12e56e2484954b891411b74dTom Talpey rc = rpcrdma_deregister_default_external(seg, ia); 1653c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ break; 1654c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1655c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (r) { 1656c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep *rep = r; 1657c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ void (*func)(struct rpcrdma_rep *) = rep->rr_func; 1658c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_func = NULL; 1659c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ func(rep); /* dereg done, callback now */ 1660c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1661c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return nsegs; 1662c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1663c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1664c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1665c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Prepost any receive buffer, then post send. 1666c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * 1667c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * Receive buffer is donated to hardware, reclaimed upon recv completion. 1668c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1669c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1670c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_post(struct rpcrdma_ia *ia, 1671c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ep *ep, 1672c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_req *req) 1673c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1674c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_send_wr send_wr, *send_wr_fail; 1675c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep *rep = req->rl_reply; 1676c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 1677c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1678c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rep) { 1679c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = rpcrdma_ep_post_recv(ia, ep, rep); 1680c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1681c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ goto out; 1682c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_reply = NULL; 1683c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1684c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1685c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.next = NULL; 1686c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.wr_id = 0ULL; /* no send cookie */ 1687c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.sg_list = req->rl_send_iov; 1688c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.num_sge = req->rl_niovs; 1689c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.opcode = IB_WR_SEND; 1690c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (send_wr.num_sge == 4) /* no need to sync any pad (constant) */ 1691c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_sync_single_for_device(ia->ri_id->device, 1692c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_send_iov[3].addr, req->rl_send_iov[3].length, 1693c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ DMA_TO_DEVICE); 1694c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_sync_single_for_device(ia->ri_id->device, 1695c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_send_iov[1].addr, req->rl_send_iov[1].length, 1696c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ DMA_TO_DEVICE); 1697c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_sync_single_for_device(ia->ri_id->device, 1698c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ req->rl_send_iov[0].addr, req->rl_send_iov[0].length, 1699c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ DMA_TO_DEVICE); 1700c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1701c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (DECR_CQCOUNT(ep) > 0) 1702c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.send_flags = 0; 1703c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ else { /* Provider must take a send completion every now and then */ 1704c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ INIT_CQCOUNT(ep); 1705c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ send_wr.send_flags = IB_SEND_SIGNALED; 1706c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ } 1707c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1708c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_post_send(ia->ri_id->qp, &send_wr, &send_wr_fail); 1709c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1710c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_post_send returned %i\n", __func__, 1711c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc); 1712c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\out: 1713c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 1714c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1715c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1716c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\/* 1717c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ * (Re)post a receive buffer. 1718c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ */ 1719c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\int 1720c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\rpcrdma_ep_post_recv(struct rpcrdma_ia *ia, 1721c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_ep *ep, 1722c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct rpcrdma_rep *rep) 1723c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\{ 1724c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ struct ib_recv_wr recv_wr, *recv_wr_fail; 1725c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ int rc; 1726c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1727c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ recv_wr.next = NULL; 1728c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ recv_wr.wr_id = (u64) (unsigned long) rep; 1729c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ recv_wr.sg_list = &rep->rr_iov; 1730c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ recv_wr.num_sge = 1; 1731c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1732c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ ib_dma_sync_single_for_cpu(ia->ri_id->device, 1733c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rep->rr_iov.addr, rep->rr_iov.length, DMA_BIDIRECTIONAL); 1734c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1735c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ DECR_CQCOUNT(ep); 1736c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc = ib_post_recv(ia->ri_id->qp, &recv_wr, &recv_wr_fail); 1737c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ 1738c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ if (rc) 1739c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ dprintk("RPC: %s: ib_post_recv returned %i\n", __func__, 1740c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ rc); 1741c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\ return rc; 1742c56c65fb67d6461f6059dd83b1750a1973a91185\"Talpey, Thomas\} 1743