15ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz/* 25ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz * videobuf2-dma-sg.c - dma scatter/gather memory allocator for videobuf2 35ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz * 45ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz * Copyright (C) 2010 Samsung Electronics 55ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz * 65ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz * Author: Andrzej Pietrasiewicz <andrzej.p@samsung.com> 75ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz * 85ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz * This program is free software; you can redistribute it and/or modify 95ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz * it under the terms of the GNU General Public License as published by 105ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz * the Free Software Foundation. 115ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz */ 125ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 135ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz#include <linux/module.h> 145ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz#include <linux/mm.h> 155ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz#include <linux/scatterlist.h> 165ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz#include <linux/sched.h> 175ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz#include <linux/slab.h> 185ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz#include <linux/vmalloc.h> 195ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 205ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz#include <media/videobuf2-core.h> 215ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz#include <media/videobuf2-memops.h> 225ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz#include <media/videobuf2-dma-sg.h> 235ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 24ffdc78efe1a8a013036aa7a63402f2708ed4f483Hans Verkuilstatic int debug; 25ffdc78efe1a8a013036aa7a63402f2708ed4f483Hans Verkuilmodule_param(debug, int, 0644); 26ffdc78efe1a8a013036aa7a63402f2708ed4f483Hans Verkuil 27ffdc78efe1a8a013036aa7a63402f2708ed4f483Hans Verkuil#define dprintk(level, fmt, arg...) \ 28ffdc78efe1a8a013036aa7a63402f2708ed4f483Hans Verkuil do { \ 29ffdc78efe1a8a013036aa7a63402f2708ed4f483Hans Verkuil if (debug >= level) \ 30ffdc78efe1a8a013036aa7a63402f2708ed4f483Hans Verkuil printk(KERN_DEBUG "vb2-dma-sg: " fmt, ## arg); \ 31ffdc78efe1a8a013036aa7a63402f2708ed4f483Hans Verkuil } while (0) 32ffdc78efe1a8a013036aa7a63402f2708ed4f483Hans Verkuil 335ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewiczstruct vb2_dma_sg_buf { 345ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz void *vaddr; 355ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz struct page **pages; 365ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz int write; 375ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz int offset; 38223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda struct sg_table sg_table; 39223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda size_t size; 40223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda unsigned int num_pages; 415ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz atomic_t refcount; 425ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz struct vb2_vmarea_handler handler; 4350ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda struct vm_area_struct *vma; 445ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz}; 455ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 465ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewiczstatic void vb2_dma_sg_put(void *buf_priv); 475ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 48df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribaldastatic int vb2_dma_sg_alloc_compacted(struct vb2_dma_sg_buf *buf, 49df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda gfp_t gfp_flags) 50df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda{ 51df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda unsigned int last_page = 0; 52223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda int size = buf->size; 53df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda 54df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda while (size > 0) { 55df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda struct page *pages; 56df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda int order; 57df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda int i; 58df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda 59df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda order = get_order(size); 60df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda /* Dont over allocate*/ 61df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda if ((PAGE_SIZE << order) > size) 62df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda order--; 63df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda 64df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda pages = NULL; 65df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda while (!pages) { 66df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda pages = alloc_pages(GFP_KERNEL | __GFP_ZERO | 67df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda __GFP_NOWARN | gfp_flags, order); 68df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda if (pages) 69df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda break; 70df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda 71df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda if (order == 0) { 72df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda while (last_page--) 73df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda __free_page(buf->pages[last_page]); 74df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda return -ENOMEM; 75df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda } 76df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda order--; 77df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda } 78df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda 79df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda split_page(pages, order); 80223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda for (i = 0; i < (1 << order); i++) 81223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda buf->pages[last_page++] = &pages[i]; 82df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda 83df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda size -= PAGE_SIZE << order; 84df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda } 85df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda 86df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda return 0; 87df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda} 88df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda 89b6ba2057f7823352bbc44ee846faa03b36e8b6acHans Verkuilstatic void *vb2_dma_sg_alloc(void *alloc_ctx, unsigned long size, gfp_t gfp_flags) 905ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz{ 915ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz struct vb2_dma_sg_buf *buf; 92df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda int ret; 93223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda int num_pages; 945ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 955ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz buf = kzalloc(sizeof *buf, GFP_KERNEL); 965ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz if (!buf) 975ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz return NULL; 985ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 995ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz buf->vaddr = NULL; 1005ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz buf->write = 0; 1015ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz buf->offset = 0; 102223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda buf->size = size; 1037f8414594e4755234fd0ca415630cfe2dfab42ceMauro Carvalho Chehab /* size is already page aligned */ 104223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda buf->num_pages = size >> PAGE_SHIFT; 1055ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 106223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda buf->pages = kzalloc(buf->num_pages * sizeof(struct page *), 1075ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz GFP_KERNEL); 1085ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz if (!buf->pages) 1095ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz goto fail_pages_array_alloc; 1105ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 111df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda ret = vb2_dma_sg_alloc_compacted(buf, gfp_flags); 112df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda if (ret) 113df23728118cd0f53070769e2ac26a255f66daa57Ricardo Ribalda goto fail_pages_alloc; 1145ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 115223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda ret = sg_alloc_table_from_pages(&buf->sg_table, buf->pages, 11647bc59c52b005f546343c373370a7eec6a2b0f84Hans Verkuil buf->num_pages, 0, size, GFP_KERNEL); 117223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda if (ret) 118223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda goto fail_table_alloc; 119223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda 1205ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz buf->handler.refcount = &buf->refcount; 1215ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz buf->handler.put = vb2_dma_sg_put; 1225ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz buf->handler.arg = buf; 1235ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 1245ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz atomic_inc(&buf->refcount); 1255ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 126ffdc78efe1a8a013036aa7a63402f2708ed4f483Hans Verkuil dprintk(1, "%s: Allocated buffer of %d pages\n", 127223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda __func__, buf->num_pages); 1285ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz return buf; 1295ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 130223012475968fb8dac866bff5b278e9311a36894Ricardo Ribaldafail_table_alloc: 131223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda num_pages = buf->num_pages; 132223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda while (num_pages--) 133223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda __free_page(buf->pages[num_pages]); 1345ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewiczfail_pages_alloc: 135a9bb36aa0d6547aa0ab3d5cfde08e8651bd460bcAndrzej Pietrasiewicz kfree(buf->pages); 1365ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewiczfail_pages_array_alloc: 1375ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz kfree(buf); 1385ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz return NULL; 1395ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz} 1405ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 1415ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewiczstatic void vb2_dma_sg_put(void *buf_priv) 1425ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz{ 1435ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz struct vb2_dma_sg_buf *buf = buf_priv; 144223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda int i = buf->num_pages; 1455ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 1465ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz if (atomic_dec_and_test(&buf->refcount)) { 147ffdc78efe1a8a013036aa7a63402f2708ed4f483Hans Verkuil dprintk(1, "%s: Freeing buffer of %d pages\n", __func__, 148223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda buf->num_pages); 1495ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz if (buf->vaddr) 150223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda vm_unmap_ram(buf->vaddr, buf->num_pages); 151223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda sg_free_table(&buf->sg_table); 1525ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz while (--i >= 0) 1535ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz __free_page(buf->pages[i]); 1545ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz kfree(buf->pages); 1555ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz kfree(buf); 1565ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz } 1575ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz} 1585ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 15950ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribaldastatic inline int vma_is_io(struct vm_area_struct *vma) 16050ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda{ 16150ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda return !!(vma->vm_flags & (VM_IO | VM_PFNMAP)); 16250ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda} 16350ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda 1645ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewiczstatic void *vb2_dma_sg_get_userptr(void *alloc_ctx, unsigned long vaddr, 1655ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz unsigned long size, int write) 1665ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz{ 1675ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz struct vb2_dma_sg_buf *buf; 1685ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz unsigned long first, last; 169223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda int num_pages_from_user; 17050ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda struct vm_area_struct *vma; 1715ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 1725ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz buf = kzalloc(sizeof *buf, GFP_KERNEL); 1735ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz if (!buf) 1745ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz return NULL; 1755ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 1765ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz buf->vaddr = NULL; 1775ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz buf->write = write; 1785ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz buf->offset = vaddr & ~PAGE_MASK; 179223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda buf->size = size; 1805ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 1815ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz first = (vaddr & PAGE_MASK) >> PAGE_SHIFT; 1825ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz last = ((vaddr + size - 1) & PAGE_MASK) >> PAGE_SHIFT; 183223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda buf->num_pages = last - first + 1; 1845ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 185223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda buf->pages = kzalloc(buf->num_pages * sizeof(struct page *), 1865ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz GFP_KERNEL); 1875ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz if (!buf->pages) 18864c832a4f79542809d6c10b8ec6225ff8b76092eGeyslan G. Bem goto userptr_fail_alloc_pages; 1895ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 19050ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda vma = find_vma(current->mm, vaddr); 19150ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda if (!vma) { 19250ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda dprintk(1, "no vma for address %lu\n", vaddr); 19350ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda goto userptr_fail_find_vma; 19450ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda } 19550ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda 19650ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda if (vma->vm_end < vaddr + size) { 19750ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda dprintk(1, "vma at %lu is too small for %lu bytes\n", 19850ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda vaddr, size); 19950ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda goto userptr_fail_find_vma; 20050ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda } 20150ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda 20250ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda buf->vma = vb2_get_vma(vma); 20350ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda if (!buf->vma) { 20450ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda dprintk(1, "failed to copy vma\n"); 20550ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda goto userptr_fail_find_vma; 20650ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda } 20750ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda 20850ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda if (vma_is_io(buf->vma)) { 20950ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda for (num_pages_from_user = 0; 21050ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda num_pages_from_user < buf->num_pages; 21150ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda ++num_pages_from_user, vaddr += PAGE_SIZE) { 21250ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda unsigned long pfn; 21350ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda 214227ae227c9352903d8bc4dc42e128da93aca4c79Ricardo Ribalda if (follow_pfn(vma, vaddr, &pfn)) { 21550ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda dprintk(1, "no page for address %lu\n", vaddr); 21650ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda break; 21750ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda } 21850ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda buf->pages[num_pages_from_user] = pfn_to_page(pfn); 21950ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda } 22050ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda } else 22150ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda num_pages_from_user = get_user_pages(current, current->mm, 2225ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz vaddr & PAGE_MASK, 223223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda buf->num_pages, 2245ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz write, 2255ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 1, /* force */ 2265ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz buf->pages, 2275ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz NULL); 228b037c0fde22b1d3cd0b3c3717d28e54619fc1592Marek Szyprowski 229223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda if (num_pages_from_user != buf->num_pages) 2305ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz goto userptr_fail_get_user_pages; 2315ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 232223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda if (sg_alloc_table_from_pages(&buf->sg_table, buf->pages, 233223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda buf->num_pages, buf->offset, size, 0)) 234223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda goto userptr_fail_alloc_table_from_pages; 235223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda 2365ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz return buf; 2375ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 238223012475968fb8dac866bff5b278e9311a36894Ricardo Ribaldauserptr_fail_alloc_table_from_pages: 2395ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewiczuserptr_fail_get_user_pages: 240ffdc78efe1a8a013036aa7a63402f2708ed4f483Hans Verkuil dprintk(1, "get_user_pages requested/got: %d/%d]\n", 241202dfbdc2b88286463b9e05df4be1c0bce50af4aRicardo Ribalda buf->num_pages, num_pages_from_user); 24250ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda if (!vma_is_io(buf->vma)) 24350ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda while (--num_pages_from_user >= 0) 24450ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda put_page(buf->pages[num_pages_from_user]); 24550ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda vb2_put_vma(buf->vma); 24650ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribaldauserptr_fail_find_vma: 247a9bb36aa0d6547aa0ab3d5cfde08e8651bd460bcAndrzej Pietrasiewicz kfree(buf->pages); 24864c832a4f79542809d6c10b8ec6225ff8b76092eGeyslan G. Bemuserptr_fail_alloc_pages: 2495ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz kfree(buf); 2505ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz return NULL; 2515ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz} 2525ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 2535ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz/* 2545ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz * @put_userptr: inform the allocator that a USERPTR buffer will no longer 2555ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz * be used 2565ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz */ 2575ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewiczstatic void vb2_dma_sg_put_userptr(void *buf_priv) 2585ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz{ 2595ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz struct vb2_dma_sg_buf *buf = buf_priv; 260223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda int i = buf->num_pages; 2615ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 262ffdc78efe1a8a013036aa7a63402f2708ed4f483Hans Verkuil dprintk(1, "%s: Releasing userspace buffer of %d pages\n", 263223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda __func__, buf->num_pages); 2645ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz if (buf->vaddr) 265223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda vm_unmap_ram(buf->vaddr, buf->num_pages); 266223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda sg_free_table(&buf->sg_table); 2675ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz while (--i >= 0) { 2685ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz if (buf->write) 2695ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz set_page_dirty_lock(buf->pages[i]); 27050ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda if (!vma_is_io(buf->vma)) 27150ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda put_page(buf->pages[i]); 2725ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz } 2735ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz kfree(buf->pages); 27450ac952d2263bd5d7812acf1839d57c34c6f8d9fRicardo Ribalda vb2_put_vma(buf->vma); 2755ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz kfree(buf); 2765ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz} 2775ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 2785ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewiczstatic void *vb2_dma_sg_vaddr(void *buf_priv) 2795ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz{ 2805ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz struct vb2_dma_sg_buf *buf = buf_priv; 2815ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 2825ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz BUG_ON(!buf); 2835ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 2845ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz if (!buf->vaddr) 2855ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz buf->vaddr = vm_map_ram(buf->pages, 286223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda buf->num_pages, 2875ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz -1, 2885ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz PAGE_KERNEL); 2895ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 2905ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz /* add offset in case userptr is not page-aligned */ 2915ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz return buf->vaddr + buf->offset; 2925ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz} 2935ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 2945ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewiczstatic unsigned int vb2_dma_sg_num_users(void *buf_priv) 2955ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz{ 2965ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz struct vb2_dma_sg_buf *buf = buf_priv; 2975ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 2985ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz return atomic_read(&buf->refcount); 2995ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz} 3005ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 3015ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewiczstatic int vb2_dma_sg_mmap(void *buf_priv, struct vm_area_struct *vma) 3025ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz{ 3035ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz struct vb2_dma_sg_buf *buf = buf_priv; 3045ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz unsigned long uaddr = vma->vm_start; 3055ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz unsigned long usize = vma->vm_end - vma->vm_start; 3065ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz int i = 0; 3075ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 3085ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz if (!buf) { 3095ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz printk(KERN_ERR "No memory to map\n"); 3105ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz return -EINVAL; 3115ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz } 3125ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 3135ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz do { 3145ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz int ret; 3155ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 3165ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz ret = vm_insert_page(vma, uaddr, buf->pages[i++]); 3175ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz if (ret) { 3185ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz printk(KERN_ERR "Remapping memory, error: %d\n", ret); 3195ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz return ret; 3205ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz } 3215ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 3225ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz uaddr += PAGE_SIZE; 3235ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz usize -= PAGE_SIZE; 3245ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz } while (usize > 0); 3255ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 3265ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 3275ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz /* 3285ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz * Use common vm_area operations to track buffer refcount. 3295ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz */ 3305ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz vma->vm_private_data = &buf->handler; 3315ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz vma->vm_ops = &vb2_common_vm_ops; 3325ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 3335ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz vma->vm_ops->open(vma); 3345ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 3355ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz return 0; 3365ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz} 3375ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 3385ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewiczstatic void *vb2_dma_sg_cookie(void *buf_priv) 3395ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz{ 3405ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz struct vb2_dma_sg_buf *buf = buf_priv; 3415ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 342223012475968fb8dac866bff5b278e9311a36894Ricardo Ribalda return &buf->sg_table; 3435ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz} 3445ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 3455ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewiczconst struct vb2_mem_ops vb2_dma_sg_memops = { 3465ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz .alloc = vb2_dma_sg_alloc, 3475ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz .put = vb2_dma_sg_put, 3485ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz .get_userptr = vb2_dma_sg_get_userptr, 3495ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz .put_userptr = vb2_dma_sg_put_userptr, 3505ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz .vaddr = vb2_dma_sg_vaddr, 3515ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz .mmap = vb2_dma_sg_mmap, 3525ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz .num_users = vb2_dma_sg_num_users, 3535ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz .cookie = vb2_dma_sg_cookie, 3545ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz}; 3555ba3f757f0592ca001266b4a6214d0332349909cAndrzej PietrasiewiczEXPORT_SYMBOL_GPL(vb2_dma_sg_memops); 3565ba3f757f0592ca001266b4a6214d0332349909cAndrzej Pietrasiewicz 3575ba3f757f0592ca001266b4a6214d0332349909cAndrzej PietrasiewiczMODULE_DESCRIPTION("dma scatter/gather memory handling routines for videobuf2"); 3585ba3f757f0592ca001266b4a6214d0332349909cAndrzej PietrasiewiczMODULE_AUTHOR("Andrzej Pietrasiewicz"); 3595ba3f757f0592ca001266b4a6214d0332349909cAndrzej PietrasiewiczMODULE_LICENSE("GPL"); 360