13d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh#include <linux/ceph/ceph_debug.h> 23d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 33d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh#include <linux/module.h> 43d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh#include <linux/sched.h> 53d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh#include <linux/slab.h> 63d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh#include <linux/file.h> 73d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh#include <linux/namei.h> 83d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh#include <linux/writeback.h> 93d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 103d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh#include <linux/ceph/libceph.h> 113d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 123d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh/* 133d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh * build a vector of user pages 143d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh */ 15b324814e8436772cb3367b14149ba003a9954525Alex Elderstruct page **ceph_get_direct_page_vector(const void __user *data, 16b6aa5901c7a2bd90d0b6b9866300d2648b2568f3Henry C Chang int num_pages, bool write_page) 173d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh{ 183d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh struct page **pages; 1938815b780285a4957852c5c9dbe94991c0b26c56Sage Weil int got = 0; 2038815b780285a4957852c5c9dbe94991c0b26c56Sage Weil int rc = 0; 213d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 223d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh pages = kmalloc(sizeof(*pages) * num_pages, GFP_NOFS); 233d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh if (!pages) 243d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh return ERR_PTR(-ENOMEM); 253d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 263d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh down_read(¤t->mm->mmap_sem); 2738815b780285a4957852c5c9dbe94991c0b26c56Sage Weil while (got < num_pages) { 2838815b780285a4957852c5c9dbe94991c0b26c56Sage Weil rc = get_user_pages(current, current->mm, 2938815b780285a4957852c5c9dbe94991c0b26c56Sage Weil (unsigned long)data + ((unsigned long)got * PAGE_SIZE), 3038815b780285a4957852c5c9dbe94991c0b26c56Sage Weil num_pages - got, write_page, 0, pages + got, NULL); 3138815b780285a4957852c5c9dbe94991c0b26c56Sage Weil if (rc < 0) 3238815b780285a4957852c5c9dbe94991c0b26c56Sage Weil break; 3338815b780285a4957852c5c9dbe94991c0b26c56Sage Weil BUG_ON(rc == 0); 3438815b780285a4957852c5c9dbe94991c0b26c56Sage Weil got += rc; 3538815b780285a4957852c5c9dbe94991c0b26c56Sage Weil } 363d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh up_read(¤t->mm->mmap_sem); 3738815b780285a4957852c5c9dbe94991c0b26c56Sage Weil if (rc < 0) 383d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh goto fail; 393d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh return pages; 403d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 413d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadehfail: 4238815b780285a4957852c5c9dbe94991c0b26c56Sage Weil ceph_put_page_vector(pages, got, false); 433d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh return ERR_PTR(rc); 443d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh} 453d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda SadehEXPORT_SYMBOL(ceph_get_direct_page_vector); 463d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 47b6aa5901c7a2bd90d0b6b9866300d2648b2568f3Henry C Changvoid ceph_put_page_vector(struct page **pages, int num_pages, bool dirty) 483d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh{ 493d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh int i; 503d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 51b6aa5901c7a2bd90d0b6b9866300d2648b2568f3Henry C Chang for (i = 0; i < num_pages; i++) { 52b6aa5901c7a2bd90d0b6b9866300d2648b2568f3Henry C Chang if (dirty) 53b6aa5901c7a2bd90d0b6b9866300d2648b2568f3Henry C Chang set_page_dirty_lock(pages[i]); 543d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh put_page(pages[i]); 55b6aa5901c7a2bd90d0b6b9866300d2648b2568f3Henry C Chang } 562b777c9dd9ebbb2f8b6818d454cc5e6d7c1e3c8bAl Viro if (is_vmalloc_addr(pages)) 572b777c9dd9ebbb2f8b6818d454cc5e6d7c1e3c8bAl Viro vfree(pages); 582b777c9dd9ebbb2f8b6818d454cc5e6d7c1e3c8bAl Viro else 592b777c9dd9ebbb2f8b6818d454cc5e6d7c1e3c8bAl Viro kfree(pages); 603d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh} 613d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda SadehEXPORT_SYMBOL(ceph_put_page_vector); 623d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 633d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadehvoid ceph_release_page_vector(struct page **pages, int num_pages) 643d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh{ 653d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh int i; 663d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 673d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh for (i = 0; i < num_pages; i++) 683d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh __free_pages(pages[i], 0); 693d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh kfree(pages); 703d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh} 713d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda SadehEXPORT_SYMBOL(ceph_release_page_vector); 723d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 733d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh/* 743d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh * allocate a vector new pages 753d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh */ 763d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadehstruct page **ceph_alloc_page_vector(int num_pages, gfp_t flags) 773d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh{ 783d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh struct page **pages; 793d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh int i; 803d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 813d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh pages = kmalloc(sizeof(*pages) * num_pages, flags); 823d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh if (!pages) 833d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh return ERR_PTR(-ENOMEM); 843d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh for (i = 0; i < num_pages; i++) { 853d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh pages[i] = __page_cache_alloc(flags); 863d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh if (pages[i] == NULL) { 873d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh ceph_release_page_vector(pages, i); 883d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh return ERR_PTR(-ENOMEM); 893d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh } 903d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh } 913d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh return pages; 923d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh} 933d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda SadehEXPORT_SYMBOL(ceph_alloc_page_vector); 943d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 953d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh/* 963d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh * copy user data into a page vector 973d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh */ 983d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadehint ceph_copy_user_to_page_vector(struct page **pages, 99b324814e8436772cb3367b14149ba003a9954525Alex Elder const void __user *data, 1003d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh loff_t off, size_t len) 1013d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh{ 1023d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh int i = 0; 1033d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh int po = off & ~PAGE_CACHE_MASK; 1043d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh int left = len; 1053d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh int l, bad; 1063d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 1073d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh while (left > 0) { 1083d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh l = min_t(int, PAGE_CACHE_SIZE-po, left); 1093d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh bad = copy_from_user(page_address(pages[i]) + po, data, l); 1103d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh if (bad == l) 1113d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh return -EFAULT; 1123d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh data += l - bad; 1133d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh left -= l - bad; 1143d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh po += l - bad; 1153d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh if (po == PAGE_CACHE_SIZE) { 1163d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh po = 0; 1173d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh i++; 1183d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh } 1193d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh } 1203d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh return len; 1213d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh} 1223d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda SadehEXPORT_SYMBOL(ceph_copy_user_to_page_vector); 1233d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 124903bb32e890237ca43ab847e561e5377cfe0fdb3Alex Eldervoid ceph_copy_to_page_vector(struct page **pages, 125b324814e8436772cb3367b14149ba003a9954525Alex Elder const void *data, 1263d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh loff_t off, size_t len) 1273d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh{ 1283d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh int i = 0; 1293d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh size_t po = off & ~PAGE_CACHE_MASK; 1303d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh size_t left = len; 1313d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 1323d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh while (left > 0) { 133903bb32e890237ca43ab847e561e5377cfe0fdb3Alex Elder size_t l = min_t(size_t, PAGE_CACHE_SIZE-po, left); 134903bb32e890237ca43ab847e561e5377cfe0fdb3Alex Elder 1353d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh memcpy(page_address(pages[i]) + po, data, l); 1363d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh data += l; 1373d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh left -= l; 1383d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh po += l; 1393d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh if (po == PAGE_CACHE_SIZE) { 1403d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh po = 0; 1413d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh i++; 1423d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh } 1433d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh } 1443d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh} 1453d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda SadehEXPORT_SYMBOL(ceph_copy_to_page_vector); 1463d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 147903bb32e890237ca43ab847e561e5377cfe0fdb3Alex Eldervoid ceph_copy_from_page_vector(struct page **pages, 148b324814e8436772cb3367b14149ba003a9954525Alex Elder void *data, 1493d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh loff_t off, size_t len) 1503d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh{ 1513d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh int i = 0; 1523d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh size_t po = off & ~PAGE_CACHE_MASK; 1533d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh size_t left = len; 1543d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 1553d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh while (left > 0) { 156903bb32e890237ca43ab847e561e5377cfe0fdb3Alex Elder size_t l = min_t(size_t, PAGE_CACHE_SIZE-po, left); 157903bb32e890237ca43ab847e561e5377cfe0fdb3Alex Elder 1583d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh memcpy(data, page_address(pages[i]) + po, l); 1593d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh data += l; 1603d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh left -= l; 1613d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh po += l; 1623d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh if (po == PAGE_CACHE_SIZE) { 1633d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh po = 0; 1643d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh i++; 1653d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh } 1663d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh } 1673d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh} 1683d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda SadehEXPORT_SYMBOL(ceph_copy_from_page_vector); 1693d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 1703d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh/* 1713d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh * Zero an extent within a page vector. Offset is relative to the 1723d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh * start of the first page. 1733d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh */ 1743d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadehvoid ceph_zero_page_vector_range(int off, int len, struct page **pages) 1753d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh{ 1763d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh int i = off >> PAGE_CACHE_SHIFT; 1773d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 1783d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh off &= ~PAGE_CACHE_MASK; 1793d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 1803d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh dout("zero_page_vector_page %u~%u\n", off, len); 1813d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 1823d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh /* leading partial page? */ 1833d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh if (off) { 1843d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh int end = min((int)PAGE_CACHE_SIZE, off + len); 1853d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh dout("zeroing %d %p head from %d\n", i, pages[i], 1863d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh (int)off); 1873d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh zero_user_segment(pages[i], off, end); 1883d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh len -= (end - off); 1893d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh i++; 1903d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh } 1913d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh while (len >= PAGE_CACHE_SIZE) { 1923d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh dout("zeroing %d %p len=%d\n", i, pages[i], len); 1933d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh zero_user_segment(pages[i], 0, PAGE_CACHE_SIZE); 1943d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh len -= PAGE_CACHE_SIZE; 1953d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh i++; 1963d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh } 1973d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh /* trailing partial page? */ 1983d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh if (len) { 1993d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh dout("zeroing %d %p tail to %d\n", i, pages[i], (int)len); 2003d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh zero_user_segment(pages[i], 0, len); 2013d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh } 2023d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh} 2033d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda SadehEXPORT_SYMBOL(ceph_zero_page_vector_range); 2043d14c5d2b6e15c21d8e5467dc62d33127c23a644Yehuda Sadeh 205