1de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech/* 2de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. 3de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * Portions based on net/core/datagram.c and copyrighted by their authors. 4de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * 5de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * This program is free software; you can redistribute it and/or modify it 6de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * under the terms of the GNU General Public License as published by the Free 7de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * Software Foundation; either version 2 of the License, or (at your option) 8de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * any later version. 9de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * 10de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * This program is distributed in the hope that it will be useful, but WITHOUT 11de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * more details. 14de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * 15de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * You should have received a copy of the GNU General Public License along with 16de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * this program; if not, write to the Free Software Foundation, Inc., 59 17de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * Temple Place - Suite 330, Boston, MA 02111-1307, USA. 18de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * 19de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * The full GNU General Public License is included in this distribution in the 20de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * file called COPYING. 21de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech */ 22de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 23de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech/* 24de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * This code allows the net stack to make use of a DMA engine for 25de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * skb to iovec copies. 26de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech */ 27de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 28de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech#include <linux/dmaengine.h> 29de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech#include <linux/pagemap.h> 305a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 31de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech#include <net/tcp.h> /* for memcpy_toiovec */ 32de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech#include <asm/io.h> 33de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech#include <asm/uaccess.h> 34de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 3556e0873b7b146564a0c36e225624f5a9b2d63791Adrian Bunkstatic int num_pages_spanned(struct iovec *iov) 36de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech{ 37de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech return 38de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech ((PAGE_ALIGN((unsigned long)iov->iov_base + iov->iov_len) - 39de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT); 40de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech} 41de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 42de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech/* 43de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * Pin down all the iovec pages needed for len bytes. 44de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * Return a struct dma_pinned_list to keep track of pages pinned down. 45de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * 46de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * We are allocating a single chunk of memory, and then carving it up into 47de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * 3 sections, the latter 2 whose size depends on the number of iovecs and the 48de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * total number of pages, respectively. 49de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech */ 50de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leechstruct dma_pinned_list *dma_pin_iovec_pages(struct iovec *iov, size_t len) 51de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech{ 52de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech struct dma_pinned_list *local_list; 53de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech struct page **pages; 54de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech int i; 55de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech int ret; 56de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech int nr_iovecs = 0; 57de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech int iovec_len_used = 0; 58de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech int iovec_pages_used = 0; 59de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 60de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech /* don't pin down non-user-based iovecs */ 61de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech if (segment_eq(get_fs(), KERNEL_DS)) 62de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech return NULL; 63de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 64de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech /* determine how many iovecs/pages there are, up front */ 65de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech do { 66de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech iovec_len_used += iov[nr_iovecs].iov_len; 67de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech iovec_pages_used += num_pages_spanned(&iov[nr_iovecs]); 68de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech nr_iovecs++; 69de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech } while (iovec_len_used < len); 70de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 71de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech /* single kmalloc for pinned list, page_list[], and the page arrays */ 72de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech local_list = kmalloc(sizeof(*local_list) 73de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech + (nr_iovecs * sizeof (struct dma_page_list)) 74de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech + (iovec_pages_used * sizeof (struct page*)), GFP_KERNEL); 75c2c0b4c5434c0a25f7f7796b29155d53805909f5Maciej Sosnowski if (!local_list) 76de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech goto out; 77de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 78de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech /* list of pages starts right after the page list array */ 79de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech pages = (struct page **) &local_list->page_list[nr_iovecs]; 80de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 81c2c0b4c5434c0a25f7f7796b29155d53805909f5Maciej Sosnowski local_list->nr_iovecs = 0; 82c2c0b4c5434c0a25f7f7796b29155d53805909f5Maciej Sosnowski 83de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech for (i = 0; i < nr_iovecs; i++) { 84de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech struct dma_page_list *page_list = &local_list->page_list[i]; 85de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 86de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech len -= iov[i].iov_len; 87de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 88c2c0b4c5434c0a25f7f7796b29155d53805909f5Maciej Sosnowski if (!access_ok(VERIFY_WRITE, iov[i].iov_base, iov[i].iov_len)) 89de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech goto unpin; 90de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 91de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech page_list->nr_pages = num_pages_spanned(&iov[i]); 92de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech page_list->base_address = iov[i].iov_base; 93de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 94de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech page_list->pages = pages; 95de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech pages += page_list->nr_pages; 96de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 97de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech /* pin pages down */ 98de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech down_read(¤t->mm->mmap_sem); 99de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech ret = get_user_pages( 100de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech current, 101de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech current->mm, 102de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech (unsigned long) iov[i].iov_base, 103de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech page_list->nr_pages, 104de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 1, /* write */ 105de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 0, /* force */ 106de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech page_list->pages, 107de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech NULL); 108de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech up_read(¤t->mm->mmap_sem); 109de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 110c2c0b4c5434c0a25f7f7796b29155d53805909f5Maciej Sosnowski if (ret != page_list->nr_pages) 111de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech goto unpin; 112de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 113de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech local_list->nr_iovecs = i + 1; 114de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech } 115de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 116de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech return local_list; 117de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 118de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leechunpin: 119de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech dma_unpin_iovec_pages(local_list); 120de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leechout: 121c2c0b4c5434c0a25f7f7796b29155d53805909f5Maciej Sosnowski return NULL; 122de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech} 123de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 124de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leechvoid dma_unpin_iovec_pages(struct dma_pinned_list *pinned_list) 125de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech{ 126de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech int i, j; 127de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 128de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech if (!pinned_list) 129de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech return; 130de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 131de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech for (i = 0; i < pinned_list->nr_iovecs; i++) { 132de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech struct dma_page_list *page_list = &pinned_list->page_list[i]; 133de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech for (j = 0; j < page_list->nr_pages; j++) { 134de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech set_page_dirty_lock(page_list->pages[j]); 135de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech page_cache_release(page_list->pages[j]); 136de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech } 137de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech } 138de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 139de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech kfree(pinned_list); 140de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech} 141de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 142de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 143de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech/* 144de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * We have already pinned down the pages we will be using in the iovecs. 145de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * Each entry in iov array has corresponding entry in pinned_list->page_list. 146de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * Using array indexing to keep iov[] and page_list[] in sync. 147de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * Initial elements in iov array's iov->iov_len will be 0 if already copied into 148de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * by another call. 149de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech * iov array length remaining guaranteed to be bigger than len. 150de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech */ 151de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leechdma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov, 152de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech struct dma_pinned_list *pinned_list, unsigned char *kdata, size_t len) 153de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech{ 154de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech int iov_byte_offset; 155de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech int copy; 156de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech dma_cookie_t dma_cookie = 0; 157de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech int iovec_idx; 158de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech int page_idx; 159de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 160de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech if (!chan) 161de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech return memcpy_toiovec(iov, kdata, len); 162de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 163de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech iovec_idx = 0; 164de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech while (iovec_idx < pinned_list->nr_iovecs) { 165de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech struct dma_page_list *page_list; 166de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 167de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech /* skip already used-up iovecs */ 168de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech while (!iov[iovec_idx].iov_len) 169de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech iovec_idx++; 170de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 171de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech page_list = &pinned_list->page_list[iovec_idx]; 172de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 173de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech iov_byte_offset = ((unsigned long)iov[iovec_idx].iov_base & ~PAGE_MASK); 174de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech page_idx = (((unsigned long)iov[iovec_idx].iov_base & PAGE_MASK) 175de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech - ((unsigned long)page_list->base_address & PAGE_MASK)) >> PAGE_SHIFT; 176de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 177de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech /* break up copies to not cross page boundary */ 178de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech while (iov[iovec_idx].iov_len) { 179de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech copy = min_t(int, PAGE_SIZE - iov_byte_offset, len); 180de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech copy = min_t(int, copy, iov[iovec_idx].iov_len); 181de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 182de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech dma_cookie = dma_async_memcpy_buf_to_pg(chan, 183de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech page_list->pages[page_idx], 184de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech iov_byte_offset, 185de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech kdata, 186de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech copy); 1874b652f0db3be891c7b76b109c3b55003b920fc96Dan Williams /* poll for a descriptor slot */ 1884b652f0db3be891c7b76b109c3b55003b920fc96Dan Williams if (unlikely(dma_cookie < 0)) { 1894b652f0db3be891c7b76b109c3b55003b920fc96Dan Williams dma_async_issue_pending(chan); 1904b652f0db3be891c7b76b109c3b55003b920fc96Dan Williams continue; 1914b652f0db3be891c7b76b109c3b55003b920fc96Dan Williams } 192de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 193de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech len -= copy; 194de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech iov[iovec_idx].iov_len -= copy; 195de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech iov[iovec_idx].iov_base += copy; 196de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 197de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech if (!len) 198de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech return dma_cookie; 199de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 200de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech kdata += copy; 201de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech iov_byte_offset = 0; 202de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech page_idx++; 203de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech } 204de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech iovec_idx++; 205de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech } 206de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 207de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech /* really bad if we ever run out of iovecs */ 208de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech BUG(); 209de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech return -EFAULT; 210de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech} 211de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 212de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leechdma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov, 213de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech struct dma_pinned_list *pinned_list, struct page *page, 214de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech unsigned int offset, size_t len) 215de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech{ 216de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech int iov_byte_offset; 217de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech int copy; 218de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech dma_cookie_t dma_cookie = 0; 219de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech int iovec_idx; 220de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech int page_idx; 221de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech int err; 222de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 223de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech /* this needs as-yet-unimplemented buf-to-buff, so punt. */ 224de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech /* TODO: use dma for this */ 225de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech if (!chan || !pinned_list) { 226de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech u8 *vaddr = kmap(page); 227de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech err = memcpy_toiovec(iov, vaddr + offset, len); 228de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech kunmap(page); 229de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech return err; 230de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech } 231de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 232de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech iovec_idx = 0; 233de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech while (iovec_idx < pinned_list->nr_iovecs) { 234de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech struct dma_page_list *page_list; 235de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 236de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech /* skip already used-up iovecs */ 237de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech while (!iov[iovec_idx].iov_len) 238de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech iovec_idx++; 239de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 240de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech page_list = &pinned_list->page_list[iovec_idx]; 241de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 242de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech iov_byte_offset = ((unsigned long)iov[iovec_idx].iov_base & ~PAGE_MASK); 243de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech page_idx = (((unsigned long)iov[iovec_idx].iov_base & PAGE_MASK) 244de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech - ((unsigned long)page_list->base_address & PAGE_MASK)) >> PAGE_SHIFT; 245de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 246de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech /* break up copies to not cross page boundary */ 247de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech while (iov[iovec_idx].iov_len) { 248de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech copy = min_t(int, PAGE_SIZE - iov_byte_offset, len); 249de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech copy = min_t(int, copy, iov[iovec_idx].iov_len); 250de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 251de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech dma_cookie = dma_async_memcpy_pg_to_pg(chan, 252de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech page_list->pages[page_idx], 253de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech iov_byte_offset, 254de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech page, 255de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech offset, 256de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech copy); 2574b652f0db3be891c7b76b109c3b55003b920fc96Dan Williams /* poll for a descriptor slot */ 2584b652f0db3be891c7b76b109c3b55003b920fc96Dan Williams if (unlikely(dma_cookie < 0)) { 2594b652f0db3be891c7b76b109c3b55003b920fc96Dan Williams dma_async_issue_pending(chan); 2604b652f0db3be891c7b76b109c3b55003b920fc96Dan Williams continue; 2614b652f0db3be891c7b76b109c3b55003b920fc96Dan Williams } 262de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 263de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech len -= copy; 264de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech iov[iovec_idx].iov_len -= copy; 265de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech iov[iovec_idx].iov_base += copy; 266de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 267de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech if (!len) 268de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech return dma_cookie; 269de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 270de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech offset += copy; 271de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech iov_byte_offset = 0; 272de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech page_idx++; 273de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech } 274de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech iovec_idx++; 275de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech } 276de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech 277de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech /* really bad if we ever run out of iovecs */ 278de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech BUG(); 279de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech return -EFAULT; 280de5506e155276d385712c2aa1c2d9a27cd4ed947Chris Leech} 281