1c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker/* 20bf4828983dff062cd502f27ab8644b32774e72eSteve Wise * Copyright (c) 2014 Open Grid Computing, Inc. All rights reserved. 3c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * Copyright (c) 2005-2006 Network Appliance, Inc. All rights reserved. 4c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * 5c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * This software is available to you under a choice of one of two 6c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * licenses. You may choose to be licensed under the terms of the GNU 7c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * General Public License (GPL) Version 2, available from the file 8c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * COPYING in the main directory of this source tree, or the BSD-type 9c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * license below: 10c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * 11c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * Redistribution and use in source and binary forms, with or without 12c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * modification, are permitted provided that the following conditions 13c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * are met: 14c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * 15c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * Redistributions of source code must retain the above copyright 16c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * notice, this list of conditions and the following disclaimer. 17c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * 18c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * Redistributions in binary form must reproduce the above 19c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * copyright notice, this list of conditions and the following 20c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * disclaimer in the documentation and/or other materials provided 21c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * with the distribution. 22c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * 23c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * Neither the name of the Network Appliance, Inc. nor the names of 24c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * its contributors may be used to endorse or promote products 25c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * derived from this software without specific prior written 26c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * permission. 27c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * 28c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 29c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 30c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 31c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 32c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 33c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 34c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 35c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 36c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 37c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 38c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 39c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * 40c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * Author: Tom Tucker <tom@opengridcomputing.com> 41c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker */ 42c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 43c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker#include <linux/sunrpc/debug.h> 44c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker#include <linux/sunrpc/rpc_rdma.h> 45c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker#include <linux/spinlock.h> 46c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker#include <asm/unaligned.h> 47c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker#include <rdma/ib_verbs.h> 48c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker#include <rdma/rdma_cm.h> 49c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker#include <linux/sunrpc/svc_rdma.h> 50c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 51c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker#define RPCDBG_FACILITY RPCDBG_SVCXPRT 52c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 53afd566ea080572499cc01d42d2f578bf4b54f20fTom Tuckerstatic int map_xdr(struct svcxprt_rdma *xprt, 54afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker struct xdr_buf *xdr, 55afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker struct svc_rdma_req_map *vec) 56c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker{ 57c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int sge_no; 58c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker u32 sge_bytes; 59c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker u32 page_bytes; 6034d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker u32 page_off; 61c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int page_no; 62c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 6334d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker BUG_ON(xdr->len != 6434d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker (xdr->head[0].iov_len + xdr->page_len + xdr->tail[0].iov_len)); 6534d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker 66c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Skip the first sge, this is for the RPCRDMA header */ 67c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker sge_no = 1; 68c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 69c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Head SGE */ 7034d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker vec->sge[sge_no].iov_base = xdr->head[0].iov_base; 7134d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker vec->sge[sge_no].iov_len = xdr->head[0].iov_len; 72c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker sge_no++; 73c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 74c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* pages SGE */ 75c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker page_no = 0; 76c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker page_bytes = xdr->page_len; 77c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker page_off = xdr->page_base; 7834d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker while (page_bytes) { 7934d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker vec->sge[sge_no].iov_base = 8034d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker page_address(xdr->pages[page_no]) + page_off; 8134d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker sge_bytes = min_t(u32, page_bytes, (PAGE_SIZE - page_off)); 82c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker page_bytes -= sge_bytes; 8334d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker vec->sge[sge_no].iov_len = sge_bytes; 84c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 85c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker sge_no++; 86c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker page_no++; 87c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker page_off = 0; /* reset for next time through loop */ 88c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 89c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 90c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Tail SGE */ 9134d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker if (xdr->tail[0].iov_len) { 9234d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker vec->sge[sge_no].iov_base = xdr->tail[0].iov_base; 9334d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker vec->sge[sge_no].iov_len = xdr->tail[0].iov_len; 94c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker sge_no++; 95c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 96c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 97b1e1e158779f1d99c2cc18e466f6bf9099fc0853Tom Talpey dprintk("svcrdma: map_xdr: sge_no %d page_no %d " 982e3c230bc7149a6af65d26a0c312e230e0c33cc3Tom Talpey "page_base %u page_len %u head_len %zu tail_len %zu\n", 99b1e1e158779f1d99c2cc18e466f6bf9099fc0853Tom Talpey sge_no, page_no, xdr->page_base, xdr->page_len, 100b1e1e158779f1d99c2cc18e466f6bf9099fc0853Tom Talpey xdr->head[0].iov_len, xdr->tail[0].iov_len); 101b1e1e158779f1d99c2cc18e466f6bf9099fc0853Tom Talpey 10234d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker vec->count = sge_no; 103afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker return 0; 104c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker} 105c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 106b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tuckerstatic dma_addr_t dma_map_xdr(struct svcxprt_rdma *xprt, 107b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker struct xdr_buf *xdr, 108b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker u32 xdr_off, size_t len, int dir) 109b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker{ 110b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker struct page *page; 111b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker dma_addr_t dma_addr; 112b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker if (xdr_off < xdr->head[0].iov_len) { 113b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker /* This offset is in the head */ 114b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker xdr_off += (unsigned long)xdr->head[0].iov_base & ~PAGE_MASK; 115b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker page = virt_to_page(xdr->head[0].iov_base); 116b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker } else { 117b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker xdr_off -= xdr->head[0].iov_len; 118b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker if (xdr_off < xdr->page_len) { 119b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker /* This offset is in the page list */ 1203cbe01a94c7b369f943f8a9d40394198d757cdd4Jeff Layton xdr_off += xdr->page_base; 121b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker page = xdr->pages[xdr_off >> PAGE_SHIFT]; 122b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker xdr_off &= ~PAGE_MASK; 123b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker } else { 124b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker /* This offset is in the tail */ 125b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker xdr_off -= xdr->page_len; 126b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker xdr_off += (unsigned long) 127b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker xdr->tail[0].iov_base & ~PAGE_MASK; 128b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker page = virt_to_page(xdr->tail[0].iov_base); 129b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker } 130b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker } 131b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker dma_addr = ib_dma_map_page(xprt->sc_cm_id->device, page, xdr_off, 132b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker min_t(size_t, PAGE_SIZE, len), dir); 133b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker return dma_addr; 134b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker} 135b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker 136c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker/* Assumptions: 137c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * - The specified write_len can be represented in sc_max_sge * PAGE_SIZE 138c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker */ 139c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tuckerstatic int send_write(struct svcxprt_rdma *xprt, struct svc_rqst *rqstp, 140c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker u32 rmr, u64 to, 141c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker u32 xdr_off, int write_len, 14234d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker struct svc_rdma_req_map *vec) 143c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker{ 144c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct ib_send_wr write_wr; 145c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct ib_sge *sge; 146c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int xdr_sge_no; 147c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int sge_no; 148c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int sge_bytes; 149c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int sge_off; 150c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int bc; 151c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct svc_rdma_op_ctxt *ctxt; 152c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 15334d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker BUG_ON(vec->count > RPCSVC_MAXPAGES); 154c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker dprintk("svcrdma: RDMA_WRITE rmr=%x, to=%llx, xdr_off=%d, " 15534d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker "write_len=%d, vec->sge=%p, vec->count=%lu\n", 156bb50c8012cbd85b8e105584b32e4d5a2d335dcefRoland Dreier rmr, (unsigned long long)to, xdr_off, 15734d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker write_len, vec->sge, vec->count); 158c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 159c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt = svc_rdma_get_context(xprt); 16034d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker ctxt->direction = DMA_TO_DEVICE; 16134d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker sge = ctxt->sge; 162c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 163c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Find the SGE associated with xdr_off */ 16434d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker for (bc = xdr_off, xdr_sge_no = 1; bc && xdr_sge_no < vec->count; 165c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker xdr_sge_no++) { 16634d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker if (vec->sge[xdr_sge_no].iov_len > bc) 167c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker break; 16834d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker bc -= vec->sge[xdr_sge_no].iov_len; 169c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 170c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 171c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker sge_off = bc; 172c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker bc = write_len; 173c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker sge_no = 0; 174c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 175c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Copy the remaining SGE */ 176afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker while (bc != 0) { 177afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker sge_bytes = min_t(size_t, 178afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker bc, vec->sge[xdr_sge_no].iov_len-sge_off); 179c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker sge[sge_no].length = sge_bytes; 1800bf4828983dff062cd502f27ab8644b32774e72eSteve Wise sge[sge_no].addr = 1810bf4828983dff062cd502f27ab8644b32774e72eSteve Wise dma_map_xdr(xprt, &rqstp->rq_res, xdr_off, 1820bf4828983dff062cd502f27ab8644b32774e72eSteve Wise sge_bytes, DMA_TO_DEVICE); 1830bf4828983dff062cd502f27ab8644b32774e72eSteve Wise xdr_off += sge_bytes; 1840bf4828983dff062cd502f27ab8644b32774e72eSteve Wise if (ib_dma_mapping_error(xprt->sc_cm_id->device, 1850bf4828983dff062cd502f27ab8644b32774e72eSteve Wise sge[sge_no].addr)) 1860bf4828983dff062cd502f27ab8644b32774e72eSteve Wise goto err; 1870bf4828983dff062cd502f27ab8644b32774e72eSteve Wise atomic_inc(&xprt->sc_dma_used); 1880bf4828983dff062cd502f27ab8644b32774e72eSteve Wise sge[sge_no].lkey = xprt->sc_dma_lkey; 189afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker ctxt->count++; 190c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker sge_off = 0; 191c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker sge_no++; 192c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker xdr_sge_no++; 193afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker BUG_ON(xdr_sge_no > vec->count); 194c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker bc -= sge_bytes; 195255942907e7ff498ab1545b5edce5690833ff640Steve Wise if (sge_no == xprt->sc_max_sge) 196255942907e7ff498ab1545b5edce5690833ff640Steve Wise break; 197c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 198c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 199c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Prepare WRITE WR */ 200c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker memset(&write_wr, 0, sizeof write_wr); 201c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt->wr_op = IB_WR_RDMA_WRITE; 202c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker write_wr.wr_id = (unsigned long)ctxt; 203c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker write_wr.sg_list = &sge[0]; 204c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker write_wr.num_sge = sge_no; 205c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker write_wr.opcode = IB_WR_RDMA_WRITE; 206c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker write_wr.send_flags = IB_SEND_SIGNALED; 207c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker write_wr.wr.rdma.rkey = rmr; 208c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker write_wr.wr.rdma.remote_addr = to; 209c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 210c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Post It */ 211c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker atomic_inc(&rdma_stat_write); 21234d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker if (svc_rdma_send(xprt, &write_wr)) 21334d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker goto err; 214255942907e7ff498ab1545b5edce5690833ff640Steve Wise return write_len - bc; 21534d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker err: 2164a84386fc27fdc7d2ea69fdbc641008e8f943159Tom Tucker svc_rdma_unmap_dma(ctxt); 21734d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker svc_rdma_put_context(ctxt, 0); 21834d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker /* Fatal error, close transport */ 21934d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker return -EIO; 220c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker} 221c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 222c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tuckerstatic int send_write_chunks(struct svcxprt_rdma *xprt, 223c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_msg *rdma_argp, 224c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_msg *rdma_resp, 225c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct svc_rqst *rqstp, 22634d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker struct svc_rdma_req_map *vec) 227c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker{ 228c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker u32 xfer_len = rqstp->rq_res.page_len + rqstp->rq_res.tail[0].iov_len; 229c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int write_len; 230c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker u32 xdr_off; 231c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int chunk_off; 232c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int chunk_no; 233c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_write_array *arg_ary; 234c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_write_array *res_ary; 235c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int ret; 236c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 237c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker arg_ary = svc_rdma_get_write_array(rdma_argp); 238c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker if (!arg_ary) 239c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker return 0; 240c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker res_ary = (struct rpcrdma_write_array *) 241c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker &rdma_resp->rm_body.rm_chunks[1]; 242c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 243c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Write chunks start at the pagelist */ 244c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker for (xdr_off = rqstp->rq_res.head[0].iov_len, chunk_no = 0; 245c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker xfer_len && chunk_no < arg_ary->wc_nchunks; 246c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker chunk_no++) { 247c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_segment *arg_ch; 248c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker u64 rs_offset; 249c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 250c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker arg_ch = &arg_ary->wc_array[chunk_no].wc_target; 251cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker write_len = min(xfer_len, ntohl(arg_ch->rs_length)); 252c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 253c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Prepare the response chunk given the length actually 254c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * written */ 255cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker xdr_decode_hyper((__be32 *)&arg_ch->rs_offset, &rs_offset); 256c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker svc_rdma_xdr_encode_array_chunk(res_ary, chunk_no, 257cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker arg_ch->rs_handle, 258cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker arg_ch->rs_offset, 259cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker write_len); 260c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker chunk_off = 0; 261c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker while (write_len) { 262c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ret = send_write(xprt, rqstp, 263cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker ntohl(arg_ch->rs_handle), 264c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker rs_offset + chunk_off, 265c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker xdr_off, 266255942907e7ff498ab1545b5edce5690833ff640Steve Wise write_len, 26734d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker vec); 268255942907e7ff498ab1545b5edce5690833ff640Steve Wise if (ret <= 0) { 269c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker dprintk("svcrdma: RDMA_WRITE failed, ret=%d\n", 270c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ret); 271c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker return -EIO; 272c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 273255942907e7ff498ab1545b5edce5690833ff640Steve Wise chunk_off += ret; 274255942907e7ff498ab1545b5edce5690833ff640Steve Wise xdr_off += ret; 275255942907e7ff498ab1545b5edce5690833ff640Steve Wise xfer_len -= ret; 276255942907e7ff498ab1545b5edce5690833ff640Steve Wise write_len -= ret; 277c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 278c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 279c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Update the req with the number of chunks actually used */ 280c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker svc_rdma_xdr_encode_write_list(rdma_resp, chunk_no); 281c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 282c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker return rqstp->rq_res.page_len + rqstp->rq_res.tail[0].iov_len; 283c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker} 284c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 285c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tuckerstatic int send_reply_chunks(struct svcxprt_rdma *xprt, 286c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_msg *rdma_argp, 287c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_msg *rdma_resp, 288c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct svc_rqst *rqstp, 28934d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker struct svc_rdma_req_map *vec) 290c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker{ 291c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker u32 xfer_len = rqstp->rq_res.len; 292c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int write_len; 293c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker u32 xdr_off; 294c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int chunk_no; 295c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int chunk_off; 296cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker int nchunks; 297c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_segment *ch; 298c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_write_array *arg_ary; 299c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_write_array *res_ary; 300c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int ret; 301c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 302c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker arg_ary = svc_rdma_get_reply_array(rdma_argp); 303c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker if (!arg_ary) 304c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker return 0; 305c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* XXX: need to fix when reply lists occur with read-list and or 306c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * write-list */ 307c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker res_ary = (struct rpcrdma_write_array *) 308c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker &rdma_resp->rm_body.rm_chunks[2]; 309c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 310c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* xdr offset starts at RPC message */ 311cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker nchunks = ntohl(arg_ary->wc_nchunks); 312c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker for (xdr_off = 0, chunk_no = 0; 313cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker xfer_len && chunk_no < nchunks; 314c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker chunk_no++) { 315c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker u64 rs_offset; 316c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ch = &arg_ary->wc_array[chunk_no].wc_target; 317cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker write_len = min(xfer_len, htonl(ch->rs_length)); 318c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 319c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Prepare the reply chunk given the length actually 320c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * written */ 321cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker xdr_decode_hyper((__be32 *)&ch->rs_offset, &rs_offset); 322c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker svc_rdma_xdr_encode_array_chunk(res_ary, chunk_no, 323cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker ch->rs_handle, ch->rs_offset, 324cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker write_len); 325c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker chunk_off = 0; 326c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker while (write_len) { 327c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ret = send_write(xprt, rqstp, 328cec56c8ff5e28f58ff13041dca7853738ae577a1Tom Tucker ntohl(ch->rs_handle), 329c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker rs_offset + chunk_off, 330c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker xdr_off, 331255942907e7ff498ab1545b5edce5690833ff640Steve Wise write_len, 33234d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker vec); 333255942907e7ff498ab1545b5edce5690833ff640Steve Wise if (ret <= 0) { 334c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker dprintk("svcrdma: RDMA_WRITE failed, ret=%d\n", 335c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ret); 336c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker return -EIO; 337c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 338255942907e7ff498ab1545b5edce5690833ff640Steve Wise chunk_off += ret; 339255942907e7ff498ab1545b5edce5690833ff640Steve Wise xdr_off += ret; 340255942907e7ff498ab1545b5edce5690833ff640Steve Wise xfer_len -= ret; 341255942907e7ff498ab1545b5edce5690833ff640Steve Wise write_len -= ret; 342c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 343c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 344c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Update the req with the number of chunks actually used */ 345c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker svc_rdma_xdr_encode_reply_array(res_ary, chunk_no); 346c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 347c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker return rqstp->rq_res.len; 348c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker} 349c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 350c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker/* This function prepares the portion of the RPCRDMA message to be 351c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * sent in the RDMA_SEND. This function is called after data sent via 352c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * RDMA has already been transmitted. There are three cases: 353c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * - The RPCRDMA header, RPC header, and payload are all sent in a 354c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * single RDMA_SEND. This is the "inline" case. 355c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * - The RPCRDMA header and some portion of the RPC header and data 356c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * are sent via this RDMA_SEND and another portion of the data is 357c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * sent via RDMA. 358c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * - The RPCRDMA header [NOMSG] is sent in this RDMA_SEND and the RPC 359c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * header and data are all transmitted via RDMA. 360c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * In all three cases, this function prepares the RPCRDMA header in 361c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * sge[0], the 'type' parameter indicates the type to place in the 362c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * RPCRDMA header, and the 'byte_count' field indicates how much of 363b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker * the XDR to include in this RDMA_SEND. NB: The offset of the payload 364b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker * to send is zero in the XDR. 365c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker */ 366c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tuckerstatic int send_reply(struct svcxprt_rdma *rdma, 367c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct svc_rqst *rqstp, 368c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct page *page, 369c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_msg *rdma_resp, 370c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct svc_rdma_op_ctxt *ctxt, 37134d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker struct svc_rdma_req_map *vec, 372c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int byte_count) 373c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker{ 374c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct ib_send_wr send_wr; 375c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int sge_no; 376c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int sge_bytes; 377c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int page_no; 378afc59400d6c65bad66d4ad0b2daf879cbff8e23eJ. Bruce Fields int pages; 379c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int ret; 380c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 3810e7f011a19696cc25d68a8d6631fc6c5aa60a54cTom Tucker /* Post a recv buffer to handle another request. */ 3820e7f011a19696cc25d68a8d6631fc6c5aa60a54cTom Tucker ret = svc_rdma_post_recv(rdma); 3830e7f011a19696cc25d68a8d6631fc6c5aa60a54cTom Tucker if (ret) { 3840e7f011a19696cc25d68a8d6631fc6c5aa60a54cTom Tucker printk(KERN_INFO 3850e7f011a19696cc25d68a8d6631fc6c5aa60a54cTom Tucker "svcrdma: could not post a receive buffer, err=%d." 3860e7f011a19696cc25d68a8d6631fc6c5aa60a54cTom Tucker "Closing transport %p.\n", ret, rdma); 3870e7f011a19696cc25d68a8d6631fc6c5aa60a54cTom Tucker set_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags); 3885ac461a6f05499fa233ea43b1de80b679d1eec21Tom Tucker svc_rdma_put_context(ctxt, 0); 3895ac461a6f05499fa233ea43b1de80b679d1eec21Tom Tucker return -ENOTCONN; 3900e7f011a19696cc25d68a8d6631fc6c5aa60a54cTom Tucker } 3910e7f011a19696cc25d68a8d6631fc6c5aa60a54cTom Tucker 392c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Prepare the context */ 393c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt->pages[0] = page; 394c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt->count = 1; 395c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 396c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Prepare the SGE for the RPCRDMA Header */ 39798779be861a05c4cb75bed916df72ec0cba8b53dSteve Wise ctxt->sge[0].lkey = rdma->sc_dma_lkey; 39898779be861a05c4cb75bed916df72ec0cba8b53dSteve Wise ctxt->sge[0].length = svc_rdma_xdr_get_reply_hdr_len(rdma_resp); 399c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt->sge[0].addr = 400b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker ib_dma_map_page(rdma->sc_cm_id->device, page, 0, 401b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker ctxt->sge[0].length, DMA_TO_DEVICE); 402afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker if (ib_dma_mapping_error(rdma->sc_cm_id->device, ctxt->sge[0].addr)) 403afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker goto err; 404afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker atomic_inc(&rdma->sc_dma_used); 405afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 406c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt->direction = DMA_TO_DEVICE; 407afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 408b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker /* Map the payload indicated by 'byte_count' */ 40934d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker for (sge_no = 1; byte_count && sge_no < vec->count; sge_no++) { 410b432e6b3d9c1b4271c43f02b45136f33a8ed5820Tom Tucker int xdr_off = 0; 41134d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker sge_bytes = min_t(size_t, vec->sge[sge_no].iov_len, byte_count); 412c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker byte_count -= sge_bytes; 4130bf4828983dff062cd502f27ab8644b32774e72eSteve Wise ctxt->sge[sge_no].addr = 4140bf4828983dff062cd502f27ab8644b32774e72eSteve Wise dma_map_xdr(rdma, &rqstp->rq_res, xdr_off, 4150bf4828983dff062cd502f27ab8644b32774e72eSteve Wise sge_bytes, DMA_TO_DEVICE); 4160bf4828983dff062cd502f27ab8644b32774e72eSteve Wise xdr_off += sge_bytes; 4170bf4828983dff062cd502f27ab8644b32774e72eSteve Wise if (ib_dma_mapping_error(rdma->sc_cm_id->device, 4180bf4828983dff062cd502f27ab8644b32774e72eSteve Wise ctxt->sge[sge_no].addr)) 4190bf4828983dff062cd502f27ab8644b32774e72eSteve Wise goto err; 4200bf4828983dff062cd502f27ab8644b32774e72eSteve Wise atomic_inc(&rdma->sc_dma_used); 4210bf4828983dff062cd502f27ab8644b32774e72eSteve Wise ctxt->sge[sge_no].lkey = rdma->sc_dma_lkey; 42234d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker ctxt->sge[sge_no].length = sge_bytes; 423c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 424c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker BUG_ON(byte_count != 0); 425c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 426c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Save all respages in the ctxt and remove them from the 427c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * respages array. They are our pages until the I/O 428c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * completes. 429c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker */ 430afc59400d6c65bad66d4ad0b2daf879cbff8e23eJ. Bruce Fields pages = rqstp->rq_next_page - rqstp->rq_respages; 431afc59400d6c65bad66d4ad0b2daf879cbff8e23eJ. Bruce Fields for (page_no = 0; page_no < pages; page_no++) { 432c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt->pages[page_no+1] = rqstp->rq_respages[page_no]; 433c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt->count++; 434c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker rqstp->rq_respages[page_no] = NULL; 435afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker /* 436afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker * If there are more pages than SGE, terminate SGE 437afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker * list so that svc_rdma_unmap_dma doesn't attempt to 438afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker * unmap garbage. 439afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker */ 44034d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker if (page_no+1 >= sge_no) 44134d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker ctxt->sge[page_no+1].length = 0; 442c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 4437e4359e2611f95a97037e2b6905eab52f28afbebTom Tucker rqstp->rq_next_page = rqstp->rq_respages + 1; 4440bf4828983dff062cd502f27ab8644b32774e72eSteve Wise 445c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker BUG_ON(sge_no > rdma->sc_max_sge); 446c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker memset(&send_wr, 0, sizeof send_wr); 447c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt->wr_op = IB_WR_SEND; 448c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker send_wr.wr_id = (unsigned long)ctxt; 449c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker send_wr.sg_list = ctxt->sge; 450c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker send_wr.num_sge = sge_no; 451c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker send_wr.opcode = IB_WR_SEND; 452c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker send_wr.send_flags = IB_SEND_SIGNALED; 453c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 454c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ret = svc_rdma_send(rdma, &send_wr); 455c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker if (ret) 456afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker goto err; 457c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 458afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker return 0; 459afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 460afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker err: 46121515e46bc6a6279dd13f6c01898ada9720100a3Steve Wise svc_rdma_unmap_dma(ctxt); 462afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker svc_rdma_put_context(ctxt, 1); 463afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker return -EIO; 464c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker} 465c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 466c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tuckervoid svc_rdma_prep_reply_hdr(struct svc_rqst *rqstp) 467c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker{ 468c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker} 469c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 470c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker/* 471c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker * Return the start of an xdr buffer. 472c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker */ 473c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tuckerstatic void *xdr_start(struct xdr_buf *xdr) 474c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker{ 475c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker return xdr->head[0].iov_base - 476c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker (xdr->len - 477c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker xdr->page_len - 478c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker xdr->tail[0].iov_len - 479c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker xdr->head[0].iov_len); 480c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker} 481c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 482c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tuckerint svc_rdma_sendto(struct svc_rqst *rqstp) 483c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker{ 484c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct svc_xprt *xprt = rqstp->rq_xprt; 485c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct svcxprt_rdma *rdma = 486c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker container_of(xprt, struct svcxprt_rdma, sc_xprt); 487c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_msg *rdma_argp; 488c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_msg *rdma_resp; 489c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct rpcrdma_write_array *reply_ary; 490c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker enum rpcrdma_proc reply_type; 491c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int ret; 492c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker int inline_bytes; 493c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct page *res_page; 494c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker struct svc_rdma_op_ctxt *ctxt; 49534d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker struct svc_rdma_req_map *vec; 496c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 497c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker dprintk("svcrdma: sending response for rqstp=%p\n", rqstp); 498c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 499c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Get the RDMA request header. */ 500c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker rdma_argp = xdr_start(&rqstp->rq_arg); 501c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 50234d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker /* Build an req vec for the XDR */ 503c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt = svc_rdma_get_context(rdma); 504c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ctxt->direction = DMA_TO_DEVICE; 50534d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker vec = svc_rdma_get_req_map(); 506afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker ret = map_xdr(rdma, &rqstp->rq_res, vec); 507afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker if (ret) 508afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker goto err0; 509c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker inline_bytes = rqstp->rq_res.len; 510c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 511c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Create the RDMA response header */ 512c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker res_page = svc_rdma_get_page(); 513c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker rdma_resp = page_address(res_page); 514c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker reply_ary = svc_rdma_get_reply_array(rdma_argp); 515c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker if (reply_ary) 516c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker reply_type = RDMA_NOMSG; 517c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker else 518c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker reply_type = RDMA_MSG; 519c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker svc_rdma_xdr_encode_reply_header(rdma, rdma_argp, 520c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker rdma_resp, reply_type); 521c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 522c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Send any write-chunk data and build resp write-list */ 523c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ret = send_write_chunks(rdma, rdma_argp, rdma_resp, 52434d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker rqstp, vec); 525c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker if (ret < 0) { 526c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker printk(KERN_ERR "svcrdma: failed to send write chunks, rc=%d\n", 527c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ret); 528afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker goto err1; 529c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 530c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker inline_bytes -= ret; 531c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 532c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker /* Send any reply-list data and update resp reply-list */ 533c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ret = send_reply_chunks(rdma, rdma_argp, rdma_resp, 53434d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker rqstp, vec); 535c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker if (ret < 0) { 536c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker printk(KERN_ERR "svcrdma: failed to send reply chunks, rc=%d\n", 537c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker ret); 538afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker goto err1; 539c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker } 540c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker inline_bytes -= ret; 541c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker 54234d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker ret = send_reply(rdma, rqstp, res_page, rdma_resp, ctxt, vec, 543c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker inline_bytes); 54434d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker svc_rdma_put_req_map(vec); 545c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker dprintk("svcrdma: send_reply returns %d\n", ret); 546c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker return ret; 547afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker 548afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker err1: 549afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker put_page(res_page); 550afd566ea080572499cc01d42d2f578bf4b54f20fTom Tucker err0: 55134d16e42a6ab74a4a4389c061dfa3c6609e08fa0Tom Tucker svc_rdma_put_req_map(vec); 552c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker svc_rdma_put_context(ctxt, 0); 553c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker return ret; 554c06b540a54ad01d2fda8cfb5d8823b9b3d8d1cb2Tom Tucker} 555