svc_rdma_sendto.c revision 7e4359e2611f95a97037e2b6905eab52f28afbeb
1c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker/* 2c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * Copyright (c) 2005-2006 Network Appliance, Inc. All rights reserved. 3c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * 4c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * This software is available to you under a choice of one of two 5c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * licenses. You may choose to be licensed under the terms of the GNU 6c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * General Public License (GPL) Version 2, available from the file 7c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * COPYING in the main directory of this source tree, or the BSD-type 8c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * license below: 9c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * 10c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * Redistribution and use in source and binary forms, with or without 11c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * modification, are permitted provided that the following conditions 12c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * are met: 13c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * 14c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * Redistributions of source code must retain the above copyright 15c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * notice, this list of conditions and the following disclaimer. 16c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * 17c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * Redistributions in binary form must reproduce the above 18c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * copyright notice, this list of conditions and the following 19c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * disclaimer in the documentation and/or other materials provided 20c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * with the distribution. 21c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * 22c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * Neither the name of the Network Appliance, Inc. nor the names of 23c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * its contributors may be used to endorse or promote products 24c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * derived from this software without specific prior written 25c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * permission. 26c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * 27c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 28c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 29c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 30c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 31c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 32c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 33c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 34c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 35c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 36c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 37c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 38c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * 39c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * Author: Tom Tucker <tom@opengridcomputing.com> 40c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker */ 41c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 42c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker#include <linux/sunrpc/debug.h> 43c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker#include <linux/sunrpc/rpc_rdma.h> 44c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker#include <linux/spinlock.h> 45c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker#include <asm/unaligned.h> 46c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker#include <rdma/ib_verbs.h> 47c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker#include <rdma/rdma_cm.h> 48c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker#include <linux/sunrpc/svc_rdma.h> 49c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 50c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker#define RPCDBG_FACILITY RPCDBG_SVCXPRT 51c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 52c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker/* Encode an XDR as an array of IB SGE 53c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * 54c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * Assumptions: 55c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * - head[0] is physically contiguous. 56c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * - tail[0] is physically contiguous. 57af901ca181d92aac3a7dc265144a9081a86d8f39André Goddard Rosa * - pages[] is not physically or virtually contiguous and consists of 58c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * PAGE_SIZE elements. 59c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * 60c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * Output: 61c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * SGE[0] reserved for RCPRDMA header 62c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * SGE[1] data from xdr->head[] 63c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * SGE[2..sge_count-2] data from xdr->pages[] 64c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * SGE[sge_count-1] data from xdr->tail. 65c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * 6634d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker * The max SGE we need is the length of the XDR / pagesize + one for 6734d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker * head + one for tail + one for RPCRDMA header. Since RPCSVC_MAXPAGES 6834d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker * reserves a page for both the request and the reply header, and this 6934d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker * array is only concerned with the reply we are assured that we have 7034d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker * on extra page for the RPCRMDA header. 71c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker */ 725eaa65b240c5eb7bf2235eb9dd177c83e6e3832cRoel Kluinstatic int fast_reg_xdr(struct svcxprt_rdma *xprt, 73b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker struct xdr_buf *xdr, 74b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker struct svc_rdma_req_map *vec) 75afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker{ 76afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker int sge_no; 77afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker u32 sge_bytes; 78afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker u32 page_bytes; 79afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker u32 page_off; 80afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker int page_no = 0; 81afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker u8 *frva; 82afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker struct svc_rdma_fastreg_mr *frmr; 83afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 84afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker frmr = svc_rdma_get_frmr(xprt); 85afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker if (IS_ERR(frmr)) 86afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker return -ENOMEM; 87afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker vec->frmr = frmr; 88afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 89afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker /* Skip the RPCRDMA header */ 90afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker sge_no = 1; 91afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 92afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker /* Map the head. */ 93afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker frva = (void *)((unsigned long)(xdr->head[0].iov_base) & PAGE_MASK); 94afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker vec->sge[sge_no].iov_base = xdr->head[0].iov_base; 95afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker vec->sge[sge_no].iov_len = xdr->head[0].iov_len; 96afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker vec->count = 2; 97afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker sge_no++; 98afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 99b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker /* Map the XDR head */ 100afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker frmr->kva = frva; 101afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker frmr->direction = DMA_TO_DEVICE; 102afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker frmr->access_flags = 0; 103afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker frmr->map_len = PAGE_SIZE; 104afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker frmr->page_list_len = 1; 105b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker page_off = (unsigned long)xdr->head[0].iov_base & ~PAGE_MASK; 106afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker frmr->page_list->page_list[page_no] = 107b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker ib_dma_map_page(xprt->sc_cm_id->device, 108b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker virt_to_page(xdr->head[0].iov_base), 109b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker page_off, 110b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker PAGE_SIZE - page_off, 111b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker DMA_TO_DEVICE); 112afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker if (ib_dma_mapping_error(xprt->sc_cm_id->device, 113afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker frmr->page_list->page_list[page_no])) 114afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker goto fatal_err; 115afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker atomic_inc(&xprt->sc_dma_used); 116afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 117b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker /* Map the XDR page list */ 118afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker page_off = xdr->page_base; 119afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker page_bytes = xdr->page_len + page_off; 120afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker if (!page_bytes) 121afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker goto encode_tail; 122afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 123afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker /* Map the pages */ 124afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker vec->sge[sge_no].iov_base = frva + frmr->map_len + page_off; 125afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker vec->sge[sge_no].iov_len = page_bytes; 126afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker sge_no++; 127afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker while (page_bytes) { 128afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker struct page *page; 129afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 130afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker page = xdr->pages[page_no++]; 131afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker sge_bytes = min_t(u32, page_bytes, (PAGE_SIZE - page_off)); 132afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker page_bytes -= sge_bytes; 133afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 134afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker frmr->page_list->page_list[page_no] = 135b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker ib_dma_map_page(xprt->sc_cm_id->device, 136b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker page, page_off, 137b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker sge_bytes, DMA_TO_DEVICE); 138afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker if (ib_dma_mapping_error(xprt->sc_cm_id->device, 139afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker frmr->page_list->page_list[page_no])) 140afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker goto fatal_err; 141afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 142afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker atomic_inc(&xprt->sc_dma_used); 143afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker page_off = 0; /* reset for next time through loop */ 144afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker frmr->map_len += PAGE_SIZE; 145afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker frmr->page_list_len++; 146afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker } 147afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker vec->count++; 148afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 149afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker encode_tail: 150afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker /* Map tail */ 151afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker if (0 == xdr->tail[0].iov_len) 152afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker goto done; 153afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 154afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker vec->count++; 155afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker vec->sge[sge_no].iov_len = xdr->tail[0].iov_len; 156afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 157afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker if (((unsigned long)xdr->tail[0].iov_base & PAGE_MASK) == 158afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker ((unsigned long)xdr->head[0].iov_base & PAGE_MASK)) { 159afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker /* 160afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker * If head and tail use the same page, we don't need 161afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker * to map it again. 162afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker */ 163afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker vec->sge[sge_no].iov_base = xdr->tail[0].iov_base; 164afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker } else { 165afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker void *va; 166afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 167afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker /* Map another page for the tail */ 168afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker page_off = (unsigned long)xdr->tail[0].iov_base & ~PAGE_MASK; 169afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker va = (void *)((unsigned long)xdr->tail[0].iov_base & PAGE_MASK); 170afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker vec->sge[sge_no].iov_base = frva + frmr->map_len + page_off; 171afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 172afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker frmr->page_list->page_list[page_no] = 173b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker ib_dma_map_page(xprt->sc_cm_id->device, virt_to_page(va), 174b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker page_off, 175b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker PAGE_SIZE, 176b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker DMA_TO_DEVICE); 177afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker if (ib_dma_mapping_error(xprt->sc_cm_id->device, 178afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker frmr->page_list->page_list[page_no])) 179afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker goto fatal_err; 180afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker atomic_inc(&xprt->sc_dma_used); 181afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker frmr->map_len += PAGE_SIZE; 182afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker frmr->page_list_len++; 183afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker } 184afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 185afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker done: 186afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker if (svc_rdma_fastreg(xprt, frmr)) 187afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker goto fatal_err; 188afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 189afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker return 0; 190afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 191afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker fatal_err: 192afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker printk("svcrdma: Error fast registering memory for xprt %p\n", xprt); 19321515e46bc6a6279dd13f6c01898ada9720100a3Steve Wise vec->frmr = NULL; 194afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker svc_rdma_put_frmr(xprt, frmr); 195afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker return -EIO; 196afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker} 197afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 198afd566ea080572499cc01d42d2f578bf4b54f20fTom Tuckerstatic int map_xdr(struct svcxprt_rdma *xprt, 199afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker struct xdr_buf *xdr, 200afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker struct svc_rdma_req_map *vec) 201c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker{ 202c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int sge_no; 203c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker u32 sge_bytes; 204c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker u32 page_bytes; 20534d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker u32 page_off; 206c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int page_no; 207c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 20834d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker BUG_ON(xdr->len != 20934d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker (xdr->head[0].iov_len + xdr->page_len + xdr->tail[0].iov_len)); 21034d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker 211afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker if (xprt->sc_frmr_pg_list_len) 212afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker return fast_reg_xdr(xprt, xdr, vec); 213afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 214c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Skip the first sge, this is for the RPCRDMA header */ 215c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker sge_no = 1; 216c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 217c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Head SGE */ 21834d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker vec->sge[sge_no].iov_base = xdr->head[0].iov_base; 21934d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker vec->sge[sge_no].iov_len = xdr->head[0].iov_len; 220c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker sge_no++; 221c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 222c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* pages SGE */ 223c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker page_no = 0; 224c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker page_bytes = xdr->page_len; 225c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker page_off = xdr->page_base; 22634d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker while (page_bytes) { 22734d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker vec->sge[sge_no].iov_base = 22834d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker page_address(xdr->pages[page_no]) + page_off; 22934d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker sge_bytes = min_t(u32, page_bytes, (PAGE_SIZE - page_off)); 230c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker page_bytes -= sge_bytes; 23134d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker vec->sge[sge_no].iov_len = sge_bytes; 232c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 233c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker sge_no++; 234c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker page_no++; 235c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker page_off = 0; /* reset for next time through loop */ 236c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 237c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 238c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Tail SGE */ 23934d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker if (xdr->tail[0].iov_len) { 24034d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker vec->sge[sge_no].iov_base = xdr->tail[0].iov_base; 24134d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker vec->sge[sge_no].iov_len = xdr->tail[0].iov_len; 242c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker sge_no++; 243c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 244c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 245b1e1e158779f1d99c2cc18e466f6bf9099fc0853Tom Talpey dprintk("svcrdma: map_xdr: sge_no %d page_no %d " 2462e3c230bc7149a6af65d26a0c312e230e0c33cc3Tom Talpey "page_base %u page_len %u head_len %zu tail_len %zu\n", 247b1e1e158779f1d99c2cc18e466f6bf9099fc0853Tom Talpey sge_no, page_no, xdr->page_base, xdr->page_len, 248b1e1e158779f1d99c2cc18e466f6bf9099fc0853Tom Talpey xdr->head[0].iov_len, xdr->tail[0].iov_len); 249b1e1e158779f1d99c2cc18e466f6bf9099fc0853Tom Talpey 25034d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker vec->count = sge_no; 251afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker return 0; 252c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker} 253c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 254b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tuckerstatic dma_addr_t dma_map_xdr(struct svcxprt_rdma *xprt, 255b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker struct xdr_buf *xdr, 256b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker u32 xdr_off, size_t len, int dir) 257b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker{ 258b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker struct page *page; 259b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker dma_addr_t dma_addr; 260b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker if (xdr_off < xdr->head[0].iov_len) { 261b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker /* This offset is in the head */ 262b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker xdr_off += (unsigned long)xdr->head[0].iov_base & ~PAGE_MASK; 263b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker page = virt_to_page(xdr->head[0].iov_base); 264b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker } else { 265b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker xdr_off -= xdr->head[0].iov_len; 266b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker if (xdr_off < xdr->page_len) { 267b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker /* This offset is in the page list */ 268b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker page = xdr->pages[xdr_off >> PAGE_SHIFT]; 269b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker xdr_off &= ~PAGE_MASK; 270b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker } else { 271b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker /* This offset is in the tail */ 272b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker xdr_off -= xdr->page_len; 273b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker xdr_off += (unsigned long) 274b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker xdr->tail[0].iov_base & ~PAGE_MASK; 275b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker page = virt_to_page(xdr->tail[0].iov_base); 276b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker } 277b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker } 278b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker dma_addr = ib_dma_map_page(xprt->sc_cm_id->device, page, xdr_off, 279b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker min_t(size_t, PAGE_SIZE, len), dir); 280b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker return dma_addr; 281b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker} 282b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker 283c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker/* Assumptions: 284afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker * - We are using FRMR 285afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker * - or - 286c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * - The specified write_len can be represented in sc_max_sge * PAGE_SIZE 287c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker */ 288c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tuckerstatic int send_write(struct svcxprt_rdma *xprt, struct svc_rqst *rqstp, 289c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker u32 rmr, u64 to, 290c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker u32 xdr_off, int write_len, 29134d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker struct svc_rdma_req_map *vec) 292c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker{ 293c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct ib_send_wr write_wr; 294c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct ib_sge *sge; 295c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int xdr_sge_no; 296c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int sge_no; 297c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int sge_bytes; 298c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int sge_off; 299c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int bc; 300c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct svc_rdma_op_ctxt *ctxt; 301c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 30234d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker BUG_ON(vec->count > RPCSVC_MAXPAGES); 303c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker dprintk("svcrdma: RDMA_WRITE rmr=%x, to=%llx, xdr_off=%d, " 30434d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker "write_len=%d, vec->sge=%p, vec->count=%lu\n", 305bb50c8012cbd85b8e105584b32e4d5a2d335dcefRoland Dreier rmr, (unsigned long long)to, xdr_off, 30634d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker write_len, vec->sge, vec->count); 307c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 308c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt = svc_rdma_get_context(xprt); 30934d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker ctxt->direction = DMA_TO_DEVICE; 31034d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker sge = ctxt->sge; 311c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 312c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Find the SGE associated with xdr_off */ 31334d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker for (bc = xdr_off, xdr_sge_no = 1; bc && xdr_sge_no < vec->count; 314c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker xdr_sge_no++) { 31534d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker if (vec->sge[xdr_sge_no].iov_len > bc) 316c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker break; 31734d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker bc -= vec->sge[xdr_sge_no].iov_len; 318c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 319c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 320c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker sge_off = bc; 321c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker bc = write_len; 322c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker sge_no = 0; 323c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 324c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Copy the remaining SGE */ 325afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker while (bc != 0) { 326afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker sge_bytes = min_t(size_t, 327afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker bc, vec->sge[xdr_sge_no].iov_len-sge_off); 328c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker sge[sge_no].length = sge_bytes; 329afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker if (!vec->frmr) { 330afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker sge[sge_no].addr = 331b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker dma_map_xdr(xprt, &rqstp->rq_res, xdr_off, 332b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker sge_bytes, DMA_TO_DEVICE); 333b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker xdr_off += sge_bytes; 334afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker if (ib_dma_mapping_error(xprt->sc_cm_id->device, 335afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker sge[sge_no].addr)) 336afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker goto err; 337afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker atomic_inc(&xprt->sc_dma_used); 338afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker sge[sge_no].lkey = xprt->sc_dma_lkey; 339afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker } else { 340afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker sge[sge_no].addr = (unsigned long) 341afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker vec->sge[xdr_sge_no].iov_base + sge_off; 342afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker sge[sge_no].lkey = vec->frmr->mr->lkey; 343afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker } 344afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker ctxt->count++; 345afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker ctxt->frmr = vec->frmr; 346c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker sge_off = 0; 347c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker sge_no++; 348c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker xdr_sge_no++; 349afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker BUG_ON(xdr_sge_no > vec->count); 350c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker bc -= sge_bytes; 351c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 352c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 353c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Prepare WRITE WR */ 354c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker memset(&write_wr, 0, sizeof write_wr); 355c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt->wr_op = IB_WR_RDMA_WRITE; 356c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker write_wr.wr_id = (unsigned long)ctxt; 357c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker write_wr.sg_list = &sge[0]; 358c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker write_wr.num_sge = sge_no; 359c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker write_wr.opcode = IB_WR_RDMA_WRITE; 360c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker write_wr.send_flags = IB_SEND_SIGNALED; 361c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker write_wr.wr.rdma.rkey = rmr; 362c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker write_wr.wr.rdma.remote_addr = to; 363c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 364c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Post It */ 365c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker atomic_inc(&rdma_stat_write); 36634d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker if (svc_rdma_send(xprt, &write_wr)) 36734d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker goto err; 36834d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker return 0; 36934d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker err: 3704a84386fc27fdc7d2ea69fdbc641008e8f943159Tom Tucker svc_rdma_unmap_dma(ctxt); 3714a84386fc27fdc7d2ea69fdbc641008e8f943159Tom Tucker svc_rdma_put_frmr(xprt, vec->frmr); 37234d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker svc_rdma_put_context(ctxt, 0); 37334d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker /* Fatal error, close transport */ 37434d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker return -EIO; 375c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker} 376c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 377c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tuckerstatic int send_write_chunks(struct svcxprt_rdma *xprt, 378c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_msg *rdma_argp, 379c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_msg *rdma_resp, 380c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct svc_rqst *rqstp, 38134d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker struct svc_rdma_req_map *vec) 382c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker{ 383c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker u32 xfer_len = rqstp->rq_res.page_len + rqstp->rq_res.tail[0].iov_len; 384c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int write_len; 385c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int max_write; 386c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker u32 xdr_off; 387c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int chunk_off; 388c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int chunk_no; 389c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_write_array *arg_ary; 390c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_write_array *res_ary; 391c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int ret; 392c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 393c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker arg_ary = svc_rdma_get_write_array(rdma_argp); 394c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker if (!arg_ary) 395c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker return 0; 396c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker res_ary = (struct rpcrdma_write_array *) 397c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker &rdma_resp->rm_body.rm_chunks[1]; 398c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 399afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker if (vec->frmr) 400afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker max_write = vec->frmr->map_len; 401afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker else 402afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker max_write = xprt->sc_max_sge * PAGE_SIZE; 403c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 404c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Write chunks start at the pagelist */ 405c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker for (xdr_off = rqstp->rq_res.head[0].iov_len, chunk_no = 0; 406c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker xfer_len && chunk_no < arg_ary->wc_nchunks; 407c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker chunk_no++) { 408c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_segment *arg_ch; 409c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker u64 rs_offset; 410c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 411c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker arg_ch = &arg_ary->wc_array[chunk_no].wc_target; 412cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker write_len = min(xfer_len, ntohl(arg_ch->rs_length)); 413c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 414c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Prepare the response chunk given the length actually 415c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * written */ 416cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker xdr_decode_hyper((__be32 *)&arg_ch->rs_offset, &rs_offset); 417c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker svc_rdma_xdr_encode_array_chunk(res_ary, chunk_no, 418cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker arg_ch->rs_handle, 419cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker arg_ch->rs_offset, 420cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker write_len); 421c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker chunk_off = 0; 422c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker while (write_len) { 423c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int this_write; 424c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker this_write = min(write_len, max_write); 425c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ret = send_write(xprt, rqstp, 426cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker ntohl(arg_ch->rs_handle), 427c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker rs_offset + chunk_off, 428c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker xdr_off, 429c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker this_write, 43034d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker vec); 431c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker if (ret) { 432c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker dprintk("svcrdma: RDMA_WRITE failed, ret=%d\n", 433c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ret); 434c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker return -EIO; 435c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 436c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker chunk_off += this_write; 437c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker xdr_off += this_write; 438c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker xfer_len -= this_write; 439c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker write_len -= this_write; 440c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 441c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 442c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Update the req with the number of chunks actually used */ 443c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker svc_rdma_xdr_encode_write_list(rdma_resp, chunk_no); 444c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 445c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker return rqstp->rq_res.page_len + rqstp->rq_res.tail[0].iov_len; 446c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker} 447c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 448c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tuckerstatic int send_reply_chunks(struct svcxprt_rdma *xprt, 449c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_msg *rdma_argp, 450c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_msg *rdma_resp, 451c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct svc_rqst *rqstp, 45234d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker struct svc_rdma_req_map *vec) 453c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker{ 454c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker u32 xfer_len = rqstp->rq_res.len; 455c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int write_len; 456c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int max_write; 457c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker u32 xdr_off; 458c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int chunk_no; 459c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int chunk_off; 460cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker int nchunks; 461c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_segment *ch; 462c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_write_array *arg_ary; 463c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_write_array *res_ary; 464c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int ret; 465c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 466c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker arg_ary = svc_rdma_get_reply_array(rdma_argp); 467c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker if (!arg_ary) 468c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker return 0; 469c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* XXX: need to fix when reply lists occur with read-list and or 470c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * write-list */ 471c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker res_ary = (struct rpcrdma_write_array *) 472c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker &rdma_resp->rm_body.rm_chunks[2]; 473c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 474afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker if (vec->frmr) 475afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker max_write = vec->frmr->map_len; 476afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker else 477afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker max_write = xprt->sc_max_sge * PAGE_SIZE; 478c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 479c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* xdr offset starts at RPC message */ 480cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker nchunks = ntohl(arg_ary->wc_nchunks); 481c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker for (xdr_off = 0, chunk_no = 0; 482cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker xfer_len && chunk_no < nchunks; 483c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker chunk_no++) { 484c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker u64 rs_offset; 485c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ch = &arg_ary->wc_array[chunk_no].wc_target; 486cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker write_len = min(xfer_len, htonl(ch->rs_length)); 487c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 488c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Prepare the reply chunk given the length actually 489c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * written */ 490cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker xdr_decode_hyper((__be32 *)&ch->rs_offset, &rs_offset); 491c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker svc_rdma_xdr_encode_array_chunk(res_ary, chunk_no, 492cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker ch->rs_handle, ch->rs_offset, 493cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker write_len); 494c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker chunk_off = 0; 495c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker while (write_len) { 496c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int this_write; 497c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 498c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker this_write = min(write_len, max_write); 499c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ret = send_write(xprt, rqstp, 500cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker ntohl(ch->rs_handle), 501c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker rs_offset + chunk_off, 502c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker xdr_off, 503c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker this_write, 50434d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker vec); 505c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker if (ret) { 506c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker dprintk("svcrdma: RDMA_WRITE failed, ret=%d\n", 507c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ret); 508c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker return -EIO; 509c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 510c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker chunk_off += this_write; 511c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker xdr_off += this_write; 512c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker xfer_len -= this_write; 513c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker write_len -= this_write; 514c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 515c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 516c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Update the req with the number of chunks actually used */ 517c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker svc_rdma_xdr_encode_reply_array(res_ary, chunk_no); 518c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 519c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker return rqstp->rq_res.len; 520c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker} 521c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 522c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker/* This function prepares the portion of the RPCRDMA message to be 523c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * sent in the RDMA_SEND. This function is called after data sent via 524c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * RDMA has already been transmitted. There are three cases: 525c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * - The RPCRDMA header, RPC header, and payload are all sent in a 526c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * single RDMA_SEND. This is the "inline" case. 527c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * - The RPCRDMA header and some portion of the RPC header and data 528c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * are sent via this RDMA_SEND and another portion of the data is 529c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * sent via RDMA. 530c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * - The RPCRDMA header [NOMSG] is sent in this RDMA_SEND and the RPC 531c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * header and data are all transmitted via RDMA. 532c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * In all three cases, this function prepares the RPCRDMA header in 533c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * sge[0], the 'type' parameter indicates the type to place in the 534c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * RPCRDMA header, and the 'byte_count' field indicates how much of 535b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker * the XDR to include in this RDMA_SEND. NB: The offset of the payload 536b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker * to send is zero in the XDR. 537c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker */ 538c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tuckerstatic int send_reply(struct svcxprt_rdma *rdma, 539c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct svc_rqst *rqstp, 540c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct page *page, 541c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_msg *rdma_resp, 542c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct svc_rdma_op_ctxt *ctxt, 54334d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker struct svc_rdma_req_map *vec, 544c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int byte_count) 545c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker{ 546c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct ib_send_wr send_wr; 547afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker struct ib_send_wr inv_wr; 548c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int sge_no; 549c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int sge_bytes; 550c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int page_no; 551afc59400d6c65bad66d4ad0b2daf879cbff8e23eJ. Bruce Fields int pages; 552c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int ret; 553c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 5540e7f011a19696cc25d68a8d6631fc6c5aa60a54cTom Tucker /* Post a recv buffer to handle another request. */ 5550e7f011a19696cc25d68a8d6631fc6c5aa60a54cTom Tucker ret = svc_rdma_post_recv(rdma); 5560e7f011a19696cc25d68a8d6631fc6c5aa60a54cTom Tucker if (ret) { 5570e7f011a19696cc25d68a8d6631fc6c5aa60a54cTom Tucker printk(KERN_INFO 5580e7f011a19696cc25d68a8d6631fc6c5aa60a54cTom Tucker "svcrdma: could not post a receive buffer, err=%d." 5590e7f011a19696cc25d68a8d6631fc6c5aa60a54cTom Tucker "Closing transport %p.\n", ret, rdma); 5600e7f011a19696cc25d68a8d6631fc6c5aa60a54cTom Tucker set_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags); 56121515e46bc6a6279dd13f6c01898ada9720100a3Steve Wise svc_rdma_put_frmr(rdma, vec->frmr); 5625ac461a6f05499fa233ea43b1de80b679d1eec21Tom Tucker svc_rdma_put_context(ctxt, 0); 5635ac461a6f05499fa233ea43b1de80b679d1eec21Tom Tucker return -ENOTCONN; 5640e7f011a19696cc25d68a8d6631fc6c5aa60a54cTom Tucker } 5650e7f011a19696cc25d68a8d6631fc6c5aa60a54cTom Tucker 566c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Prepare the context */ 567c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt->pages[0] = page; 568c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt->count = 1; 569afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker ctxt->frmr = vec->frmr; 570afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker if (vec->frmr) 571afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker set_bit(RDMACTXT_F_FAST_UNREG, &ctxt->flags); 572afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker else 573afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker clear_bit(RDMACTXT_F_FAST_UNREG, &ctxt->flags); 574c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 575c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Prepare the SGE for the RPCRDMA Header */ 57698779be861a05c4cb75bed916df72ec0cba8b53dSteve Wise ctxt->sge[0].lkey = rdma->sc_dma_lkey; 57798779be861a05c4cb75bed916df72ec0cba8b53dSteve Wise ctxt->sge[0].length = svc_rdma_xdr_get_reply_hdr_len(rdma_resp); 578c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt->sge[0].addr = 579b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker ib_dma_map_page(rdma->sc_cm_id->device, page, 0, 580b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker ctxt->sge[0].length, DMA_TO_DEVICE); 581afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker if (ib_dma_mapping_error(rdma->sc_cm_id->device, ctxt->sge[0].addr)) 582afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker goto err; 583afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker atomic_inc(&rdma->sc_dma_used); 584afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 585c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt->direction = DMA_TO_DEVICE; 586afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 587b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker /* Map the payload indicated by 'byte_count' */ 58834d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker for (sge_no = 1; byte_count && sge_no < vec->count; sge_no++) { 589b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker int xdr_off = 0; 59034d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker sge_bytes = min_t(size_t, vec->sge[sge_no].iov_len, byte_count); 591c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker byte_count -= sge_bytes; 592afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker if (!vec->frmr) { 593afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker ctxt->sge[sge_no].addr = 594b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker dma_map_xdr(rdma, &rqstp->rq_res, xdr_off, 595b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker sge_bytes, DMA_TO_DEVICE); 596b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker xdr_off += sge_bytes; 597afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker if (ib_dma_mapping_error(rdma->sc_cm_id->device, 598afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker ctxt->sge[sge_no].addr)) 599afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker goto err; 600afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker atomic_inc(&rdma->sc_dma_used); 601afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker ctxt->sge[sge_no].lkey = rdma->sc_dma_lkey; 602afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker } else { 603afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker ctxt->sge[sge_no].addr = (unsigned long) 604afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker vec->sge[sge_no].iov_base; 605afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker ctxt->sge[sge_no].lkey = vec->frmr->mr->lkey; 606afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker } 60734d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker ctxt->sge[sge_no].length = sge_bytes; 608c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 609c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker BUG_ON(byte_count != 0); 610c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 611c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Save all respages in the ctxt and remove them from the 612c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * respages array. They are our pages until the I/O 613c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * completes. 614c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker */ 615afc59400d6c65bad66d4ad0b2daf879cbff8e23eJ. Bruce Fields pages = rqstp->rq_next_page - rqstp->rq_respages; 616afc59400d6c65bad66d4ad0b2daf879cbff8e23eJ. Bruce Fields for (page_no = 0; page_no < pages; page_no++) { 617c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt->pages[page_no+1] = rqstp->rq_respages[page_no]; 618c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt->count++; 619c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker rqstp->rq_respages[page_no] = NULL; 620afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker /* 621afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker * If there are more pages than SGE, terminate SGE 622afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker * list so that svc_rdma_unmap_dma doesn't attempt to 623afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker * unmap garbage. 624afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker */ 62534d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker if (page_no+1 >= sge_no) 62634d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker ctxt->sge[page_no+1].length = 0; 627c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 6287e4359e2611f95a97037e2b6905eab52f28afbebTom Tucker rqstp->rq_next_page = rqstp->rq_respages + 1; 629c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker BUG_ON(sge_no > rdma->sc_max_sge); 630c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker memset(&send_wr, 0, sizeof send_wr); 631c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt->wr_op = IB_WR_SEND; 632c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker send_wr.wr_id = (unsigned long)ctxt; 633c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker send_wr.sg_list = ctxt->sge; 634c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker send_wr.num_sge = sge_no; 635c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker send_wr.opcode = IB_WR_SEND; 636c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker send_wr.send_flags = IB_SEND_SIGNALED; 637afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker if (vec->frmr) { 638afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker /* Prepare INVALIDATE WR */ 639afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker memset(&inv_wr, 0, sizeof inv_wr); 640afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker inv_wr.opcode = IB_WR_LOCAL_INV; 641afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker inv_wr.send_flags = IB_SEND_SIGNALED; 642afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker inv_wr.ex.invalidate_rkey = 643afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker vec->frmr->mr->lkey; 644afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker send_wr.next = &inv_wr; 645afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker } 646c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 647c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ret = svc_rdma_send(rdma, &send_wr); 648c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker if (ret) 649afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker goto err; 650c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 651afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker return 0; 652afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 653afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker err: 65421515e46bc6a6279dd13f6c01898ada9720100a3Steve Wise svc_rdma_unmap_dma(ctxt); 655afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker svc_rdma_put_frmr(rdma, vec->frmr); 656afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker svc_rdma_put_context(ctxt, 1); 657afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker return -EIO; 658c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker} 659c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 660c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tuckervoid svc_rdma_prep_reply_hdr(struct svc_rqst *rqstp) 661c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker{ 662c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker} 663c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 664c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker/* 665c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * Return the start of an xdr buffer. 666c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker */ 667c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tuckerstatic void *xdr_start(struct xdr_buf *xdr) 668c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker{ 669c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker return xdr->head[0].iov_base - 670c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker (xdr->len - 671c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker xdr->page_len - 672c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker xdr->tail[0].iov_len - 673c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker xdr->head[0].iov_len); 674c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker} 675c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 676c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tuckerint svc_rdma_sendto(struct svc_rqst *rqstp) 677c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker{ 678c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct svc_xprt *xprt = rqstp->rq_xprt; 679c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct svcxprt_rdma *rdma = 680c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker container_of(xprt, struct svcxprt_rdma, sc_xprt); 681c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_msg *rdma_argp; 682c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_msg *rdma_resp; 683c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_write_array *reply_ary; 684c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker enum rpcrdma_proc reply_type; 685c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int ret; 686c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int inline_bytes; 687c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct page *res_page; 688c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct svc_rdma_op_ctxt *ctxt; 68934d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker struct svc_rdma_req_map *vec; 690c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 691c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker dprintk("svcrdma: sending response for rqstp=%p\n", rqstp); 692c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 693c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Get the RDMA request header. */ 694c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker rdma_argp = xdr_start(&rqstp->rq_arg); 695c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 69634d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker /* Build an req vec for the XDR */ 697c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt = svc_rdma_get_context(rdma); 698c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt->direction = DMA_TO_DEVICE; 69934d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker vec = svc_rdma_get_req_map(); 700afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker ret = map_xdr(rdma, &rqstp->rq_res, vec); 701afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker if (ret) 702afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker goto err0; 703c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker inline_bytes = rqstp->rq_res.len; 704c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 705c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Create the RDMA response header */ 706c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker res_page = svc_rdma_get_page(); 707c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker rdma_resp = page_address(res_page); 708c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker reply_ary = svc_rdma_get_reply_array(rdma_argp); 709c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker if (reply_ary) 710c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker reply_type = RDMA_NOMSG; 711c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker else 712c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker reply_type = RDMA_MSG; 713c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker svc_rdma_xdr_encode_reply_header(rdma, rdma_argp, 714c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker rdma_resp, reply_type); 715c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 716c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Send any write-chunk data and build resp write-list */ 717c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ret = send_write_chunks(rdma, rdma_argp, rdma_resp, 71834d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker rqstp, vec); 719c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker if (ret < 0) { 720c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker printk(KERN_ERR "svcrdma: failed to send write chunks, rc=%d\n", 721c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ret); 722afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker goto err1; 723c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 724c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker inline_bytes -= ret; 725c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 726c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Send any reply-list data and update resp reply-list */ 727c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ret = send_reply_chunks(rdma, rdma_argp, rdma_resp, 72834d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker rqstp, vec); 729c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker if (ret < 0) { 730c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker printk(KERN_ERR "svcrdma: failed to send reply chunks, rc=%d\n", 731c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ret); 732afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker goto err1; 733c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 734c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker inline_bytes -= ret; 735c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 73634d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker ret = send_reply(rdma, rqstp, res_page, rdma_resp, ctxt, vec, 737c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker inline_bytes); 73834d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker svc_rdma_put_req_map(vec); 739c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker dprintk("svcrdma: send_reply returns %d\n", ret); 740c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker return ret; 741afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 742afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker err1: 743afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker put_page(res_page); 744afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker err0: 74534d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker svc_rdma_put_req_map(vec); 746c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker svc_rdma_put_context(ctxt, 0); 747c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker return ret; 748c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker} 749