14f18cd317a118c28482f97303600a2fe2ada6c79Al Viro#include <linux/export.h> 24f18cd317a118c28482f97303600a2fe2ada6c79Al Viro#include <linux/uio.h> 34f18cd317a118c28482f97303600a2fe2ada6c79Al Viro#include <linux/pagemap.h> 491f79c43d1b54d7154b118860d81b39bad07dfffAl Viro#include <linux/slab.h> 591f79c43d1b54d7154b118860d81b39bad07dfffAl Viro#include <linux/vmalloc.h> 64f18cd317a118c28482f97303600a2fe2ada6c79Al Viro 7c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcoxstatic size_t copy_to_iter_iovec(void *from, size_t bytes, struct iov_iter *i) 8c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox{ 9c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox size_t skip, copy, left, wanted; 10c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox const struct iovec *iov; 11c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox char __user *buf; 12c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 13c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox if (unlikely(bytes > i->count)) 14c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox bytes = i->count; 15c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 16c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox if (unlikely(!bytes)) 17c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox return 0; 18c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 19c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox wanted = bytes; 20c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox iov = i->iov; 21c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox skip = i->iov_offset; 22c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox buf = iov->iov_base + skip; 23c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox copy = min(bytes, iov->iov_len - skip); 24c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 25c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox left = __copy_to_user(buf, from, copy); 26c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox copy -= left; 27c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox skip += copy; 28c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox from += copy; 29c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox bytes -= copy; 30c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox while (unlikely(!left && bytes)) { 31c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox iov++; 32c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox buf = iov->iov_base; 33c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox copy = min(bytes, iov->iov_len); 34c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox left = __copy_to_user(buf, from, copy); 35c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox copy -= left; 36c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox skip = copy; 37c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox from += copy; 38c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox bytes -= copy; 39c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox } 40c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 41c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox if (skip == iov->iov_len) { 42c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox iov++; 43c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox skip = 0; 44c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox } 45c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox i->count -= wanted - bytes; 46c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox i->nr_segs -= iov - i->iov; 47c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox i->iov = iov; 48c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox i->iov_offset = skip; 49c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox return wanted - bytes; 50c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox} 51c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 52c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcoxstatic size_t copy_from_iter_iovec(void *to, size_t bytes, struct iov_iter *i) 53c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox{ 54c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox size_t skip, copy, left, wanted; 55c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox const struct iovec *iov; 56c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox char __user *buf; 57c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 58c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox if (unlikely(bytes > i->count)) 59c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox bytes = i->count; 60c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 61c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox if (unlikely(!bytes)) 62c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox return 0; 63c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 64c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox wanted = bytes; 65c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox iov = i->iov; 66c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox skip = i->iov_offset; 67c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox buf = iov->iov_base + skip; 68c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox copy = min(bytes, iov->iov_len - skip); 69c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 70c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox left = __copy_from_user(to, buf, copy); 71c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox copy -= left; 72c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox skip += copy; 73c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox to += copy; 74c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox bytes -= copy; 75c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox while (unlikely(!left && bytes)) { 76c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox iov++; 77c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox buf = iov->iov_base; 78c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox copy = min(bytes, iov->iov_len); 79c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox left = __copy_from_user(to, buf, copy); 80c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox copy -= left; 81c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox skip = copy; 82c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox to += copy; 83c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox bytes -= copy; 84c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox } 85c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 86c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox if (skip == iov->iov_len) { 87c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox iov++; 88c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox skip = 0; 89c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox } 90c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox i->count -= wanted - bytes; 91c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox i->nr_segs -= iov - i->iov; 92c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox i->iov = iov; 93c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox i->iov_offset = skip; 94c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox return wanted - bytes; 95c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox} 96c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 9762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virostatic size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes, 984f18cd317a118c28482f97303600a2fe2ada6c79Al Viro struct iov_iter *i) 994f18cd317a118c28482f97303600a2fe2ada6c79Al Viro{ 1004f18cd317a118c28482f97303600a2fe2ada6c79Al Viro size_t skip, copy, left, wanted; 1014f18cd317a118c28482f97303600a2fe2ada6c79Al Viro const struct iovec *iov; 1024f18cd317a118c28482f97303600a2fe2ada6c79Al Viro char __user *buf; 1034f18cd317a118c28482f97303600a2fe2ada6c79Al Viro void *kaddr, *from; 1044f18cd317a118c28482f97303600a2fe2ada6c79Al Viro 1054f18cd317a118c28482f97303600a2fe2ada6c79Al Viro if (unlikely(bytes > i->count)) 1064f18cd317a118c28482f97303600a2fe2ada6c79Al Viro bytes = i->count; 1074f18cd317a118c28482f97303600a2fe2ada6c79Al Viro 1084f18cd317a118c28482f97303600a2fe2ada6c79Al Viro if (unlikely(!bytes)) 1094f18cd317a118c28482f97303600a2fe2ada6c79Al Viro return 0; 1104f18cd317a118c28482f97303600a2fe2ada6c79Al Viro 1114f18cd317a118c28482f97303600a2fe2ada6c79Al Viro wanted = bytes; 1124f18cd317a118c28482f97303600a2fe2ada6c79Al Viro iov = i->iov; 1134f18cd317a118c28482f97303600a2fe2ada6c79Al Viro skip = i->iov_offset; 1144f18cd317a118c28482f97303600a2fe2ada6c79Al Viro buf = iov->iov_base + skip; 1154f18cd317a118c28482f97303600a2fe2ada6c79Al Viro copy = min(bytes, iov->iov_len - skip); 1164f18cd317a118c28482f97303600a2fe2ada6c79Al Viro 1174f18cd317a118c28482f97303600a2fe2ada6c79Al Viro if (!fault_in_pages_writeable(buf, copy)) { 1184f18cd317a118c28482f97303600a2fe2ada6c79Al Viro kaddr = kmap_atomic(page); 1194f18cd317a118c28482f97303600a2fe2ada6c79Al Viro from = kaddr + offset; 1204f18cd317a118c28482f97303600a2fe2ada6c79Al Viro 1214f18cd317a118c28482f97303600a2fe2ada6c79Al Viro /* first chunk, usually the only one */ 1224f18cd317a118c28482f97303600a2fe2ada6c79Al Viro left = __copy_to_user_inatomic(buf, from, copy); 1234f18cd317a118c28482f97303600a2fe2ada6c79Al Viro copy -= left; 1244f18cd317a118c28482f97303600a2fe2ada6c79Al Viro skip += copy; 1254f18cd317a118c28482f97303600a2fe2ada6c79Al Viro from += copy; 1264f18cd317a118c28482f97303600a2fe2ada6c79Al Viro bytes -= copy; 1274f18cd317a118c28482f97303600a2fe2ada6c79Al Viro 1284f18cd317a118c28482f97303600a2fe2ada6c79Al Viro while (unlikely(!left && bytes)) { 1294f18cd317a118c28482f97303600a2fe2ada6c79Al Viro iov++; 1304f18cd317a118c28482f97303600a2fe2ada6c79Al Viro buf = iov->iov_base; 1314f18cd317a118c28482f97303600a2fe2ada6c79Al Viro copy = min(bytes, iov->iov_len); 1324f18cd317a118c28482f97303600a2fe2ada6c79Al Viro left = __copy_to_user_inatomic(buf, from, copy); 1334f18cd317a118c28482f97303600a2fe2ada6c79Al Viro copy -= left; 1344f18cd317a118c28482f97303600a2fe2ada6c79Al Viro skip = copy; 1354f18cd317a118c28482f97303600a2fe2ada6c79Al Viro from += copy; 1364f18cd317a118c28482f97303600a2fe2ada6c79Al Viro bytes -= copy; 1374f18cd317a118c28482f97303600a2fe2ada6c79Al Viro } 1384f18cd317a118c28482f97303600a2fe2ada6c79Al Viro if (likely(!bytes)) { 1394f18cd317a118c28482f97303600a2fe2ada6c79Al Viro kunmap_atomic(kaddr); 1404f18cd317a118c28482f97303600a2fe2ada6c79Al Viro goto done; 1414f18cd317a118c28482f97303600a2fe2ada6c79Al Viro } 1424f18cd317a118c28482f97303600a2fe2ada6c79Al Viro offset = from - kaddr; 1434f18cd317a118c28482f97303600a2fe2ada6c79Al Viro buf += copy; 1444f18cd317a118c28482f97303600a2fe2ada6c79Al Viro kunmap_atomic(kaddr); 1454f18cd317a118c28482f97303600a2fe2ada6c79Al Viro copy = min(bytes, iov->iov_len - skip); 1464f18cd317a118c28482f97303600a2fe2ada6c79Al Viro } 1474f18cd317a118c28482f97303600a2fe2ada6c79Al Viro /* Too bad - revert to non-atomic kmap */ 1484f18cd317a118c28482f97303600a2fe2ada6c79Al Viro kaddr = kmap(page); 1494f18cd317a118c28482f97303600a2fe2ada6c79Al Viro from = kaddr + offset; 1504f18cd317a118c28482f97303600a2fe2ada6c79Al Viro left = __copy_to_user(buf, from, copy); 1514f18cd317a118c28482f97303600a2fe2ada6c79Al Viro copy -= left; 1524f18cd317a118c28482f97303600a2fe2ada6c79Al Viro skip += copy; 1534f18cd317a118c28482f97303600a2fe2ada6c79Al Viro from += copy; 1544f18cd317a118c28482f97303600a2fe2ada6c79Al Viro bytes -= copy; 1554f18cd317a118c28482f97303600a2fe2ada6c79Al Viro while (unlikely(!left && bytes)) { 1564f18cd317a118c28482f97303600a2fe2ada6c79Al Viro iov++; 1574f18cd317a118c28482f97303600a2fe2ada6c79Al Viro buf = iov->iov_base; 1584f18cd317a118c28482f97303600a2fe2ada6c79Al Viro copy = min(bytes, iov->iov_len); 1594f18cd317a118c28482f97303600a2fe2ada6c79Al Viro left = __copy_to_user(buf, from, copy); 1604f18cd317a118c28482f97303600a2fe2ada6c79Al Viro copy -= left; 1614f18cd317a118c28482f97303600a2fe2ada6c79Al Viro skip = copy; 1624f18cd317a118c28482f97303600a2fe2ada6c79Al Viro from += copy; 1634f18cd317a118c28482f97303600a2fe2ada6c79Al Viro bytes -= copy; 1644f18cd317a118c28482f97303600a2fe2ada6c79Al Viro } 1654f18cd317a118c28482f97303600a2fe2ada6c79Al Viro kunmap(page); 1664f18cd317a118c28482f97303600a2fe2ada6c79Al Virodone: 16781055e584f9d743cb13dc7944923d817c20f089dAl Viro if (skip == iov->iov_len) { 16881055e584f9d743cb13dc7944923d817c20f089dAl Viro iov++; 16981055e584f9d743cb13dc7944923d817c20f089dAl Viro skip = 0; 17081055e584f9d743cb13dc7944923d817c20f089dAl Viro } 1714f18cd317a118c28482f97303600a2fe2ada6c79Al Viro i->count -= wanted - bytes; 1724f18cd317a118c28482f97303600a2fe2ada6c79Al Viro i->nr_segs -= iov - i->iov; 1734f18cd317a118c28482f97303600a2fe2ada6c79Al Viro i->iov = iov; 1744f18cd317a118c28482f97303600a2fe2ada6c79Al Viro i->iov_offset = skip; 1754f18cd317a118c28482f97303600a2fe2ada6c79Al Viro return wanted - bytes; 1764f18cd317a118c28482f97303600a2fe2ada6c79Al Viro} 1774f18cd317a118c28482f97303600a2fe2ada6c79Al Viro 17862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virostatic size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes, 179f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro struct iov_iter *i) 180f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro{ 181f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro size_t skip, copy, left, wanted; 182f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro const struct iovec *iov; 183f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro char __user *buf; 184f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro void *kaddr, *to; 185f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro 186f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro if (unlikely(bytes > i->count)) 187f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro bytes = i->count; 188f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro 189f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro if (unlikely(!bytes)) 190f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro return 0; 191f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro 192f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro wanted = bytes; 193f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro iov = i->iov; 194f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro skip = i->iov_offset; 195f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro buf = iov->iov_base + skip; 196f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro copy = min(bytes, iov->iov_len - skip); 197f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro 198f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro if (!fault_in_pages_readable(buf, copy)) { 199f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro kaddr = kmap_atomic(page); 200f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro to = kaddr + offset; 201f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro 202f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro /* first chunk, usually the only one */ 203f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro left = __copy_from_user_inatomic(to, buf, copy); 204f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro copy -= left; 205f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro skip += copy; 206f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro to += copy; 207f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro bytes -= copy; 208f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro 209f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro while (unlikely(!left && bytes)) { 210f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro iov++; 211f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro buf = iov->iov_base; 212f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro copy = min(bytes, iov->iov_len); 213f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro left = __copy_from_user_inatomic(to, buf, copy); 214f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro copy -= left; 215f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro skip = copy; 216f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro to += copy; 217f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro bytes -= copy; 218f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro } 219f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro if (likely(!bytes)) { 220f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro kunmap_atomic(kaddr); 221f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro goto done; 222f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro } 223f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro offset = to - kaddr; 224f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro buf += copy; 225f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro kunmap_atomic(kaddr); 226f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro copy = min(bytes, iov->iov_len - skip); 227f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro } 228f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro /* Too bad - revert to non-atomic kmap */ 229f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro kaddr = kmap(page); 230f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro to = kaddr + offset; 231f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro left = __copy_from_user(to, buf, copy); 232f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro copy -= left; 233f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro skip += copy; 234f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro to += copy; 235f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro bytes -= copy; 236f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro while (unlikely(!left && bytes)) { 237f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro iov++; 238f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro buf = iov->iov_base; 239f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro copy = min(bytes, iov->iov_len); 240f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro left = __copy_from_user(to, buf, copy); 241f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro copy -= left; 242f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro skip = copy; 243f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro to += copy; 244f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro bytes -= copy; 245f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro } 246f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro kunmap(page); 247f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Virodone: 24881055e584f9d743cb13dc7944923d817c20f089dAl Viro if (skip == iov->iov_len) { 24981055e584f9d743cb13dc7944923d817c20f089dAl Viro iov++; 25081055e584f9d743cb13dc7944923d817c20f089dAl Viro skip = 0; 25181055e584f9d743cb13dc7944923d817c20f089dAl Viro } 252f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro i->count -= wanted - bytes; 253f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro i->nr_segs -= iov - i->iov; 254f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro i->iov = iov; 255f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro i->iov_offset = skip; 256f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro return wanted - bytes; 257f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro} 258f0d1bec9d58d4c038d0ac958c9af82be6eb18045Al Viro 259c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcoxstatic size_t zero_iovec(size_t bytes, struct iov_iter *i) 260c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox{ 261c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox size_t skip, copy, left, wanted; 262c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox const struct iovec *iov; 263c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox char __user *buf; 264c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 265c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox if (unlikely(bytes > i->count)) 266c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox bytes = i->count; 267c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 268c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox if (unlikely(!bytes)) 269c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox return 0; 270c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 271c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox wanted = bytes; 272c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox iov = i->iov; 273c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox skip = i->iov_offset; 274c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox buf = iov->iov_base + skip; 275c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox copy = min(bytes, iov->iov_len - skip); 276c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 277c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox left = __clear_user(buf, copy); 278c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox copy -= left; 279c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox skip += copy; 280c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox bytes -= copy; 281c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 282c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox while (unlikely(!left && bytes)) { 283c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox iov++; 284c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox buf = iov->iov_base; 285c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox copy = min(bytes, iov->iov_len); 286c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox left = __clear_user(buf, copy); 287c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox copy -= left; 288c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox skip = copy; 289c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox bytes -= copy; 290c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox } 291c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 292c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox if (skip == iov->iov_len) { 293c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox iov++; 294c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox skip = 0; 295c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox } 296c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox i->count -= wanted - bytes; 297c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox i->nr_segs -= iov - i->iov; 298c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox i->iov = iov; 299c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox i->iov_offset = skip; 300c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox return wanted - bytes; 301c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox} 302c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 3034f18cd317a118c28482f97303600a2fe2ada6c79Al Virostatic size_t __iovec_copy_from_user_inatomic(char *vaddr, 3044f18cd317a118c28482f97303600a2fe2ada6c79Al Viro const struct iovec *iov, size_t base, size_t bytes) 3054f18cd317a118c28482f97303600a2fe2ada6c79Al Viro{ 3064f18cd317a118c28482f97303600a2fe2ada6c79Al Viro size_t copied = 0, left = 0; 3074f18cd317a118c28482f97303600a2fe2ada6c79Al Viro 3084f18cd317a118c28482f97303600a2fe2ada6c79Al Viro while (bytes) { 3094f18cd317a118c28482f97303600a2fe2ada6c79Al Viro char __user *buf = iov->iov_base + base; 3104f18cd317a118c28482f97303600a2fe2ada6c79Al Viro int copy = min(bytes, iov->iov_len - base); 3114f18cd317a118c28482f97303600a2fe2ada6c79Al Viro 3124f18cd317a118c28482f97303600a2fe2ada6c79Al Viro base = 0; 3134f18cd317a118c28482f97303600a2fe2ada6c79Al Viro left = __copy_from_user_inatomic(vaddr, buf, copy); 3144f18cd317a118c28482f97303600a2fe2ada6c79Al Viro copied += copy; 3154f18cd317a118c28482f97303600a2fe2ada6c79Al Viro bytes -= copy; 3164f18cd317a118c28482f97303600a2fe2ada6c79Al Viro vaddr += copy; 3174f18cd317a118c28482f97303600a2fe2ada6c79Al Viro iov++; 3184f18cd317a118c28482f97303600a2fe2ada6c79Al Viro 3194f18cd317a118c28482f97303600a2fe2ada6c79Al Viro if (unlikely(left)) 3204f18cd317a118c28482f97303600a2fe2ada6c79Al Viro break; 3214f18cd317a118c28482f97303600a2fe2ada6c79Al Viro } 3224f18cd317a118c28482f97303600a2fe2ada6c79Al Viro return copied - left; 3234f18cd317a118c28482f97303600a2fe2ada6c79Al Viro} 3244f18cd317a118c28482f97303600a2fe2ada6c79Al Viro 3254f18cd317a118c28482f97303600a2fe2ada6c79Al Viro/* 3264f18cd317a118c28482f97303600a2fe2ada6c79Al Viro * Copy as much as we can into the page and return the number of bytes which 3274f18cd317a118c28482f97303600a2fe2ada6c79Al Viro * were successfully copied. If a fault is encountered then return the number of 3284f18cd317a118c28482f97303600a2fe2ada6c79Al Viro * bytes which were copied. 3294f18cd317a118c28482f97303600a2fe2ada6c79Al Viro */ 33062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virostatic size_t copy_from_user_atomic_iovec(struct page *page, 3314f18cd317a118c28482f97303600a2fe2ada6c79Al Viro struct iov_iter *i, unsigned long offset, size_t bytes) 3324f18cd317a118c28482f97303600a2fe2ada6c79Al Viro{ 3334f18cd317a118c28482f97303600a2fe2ada6c79Al Viro char *kaddr; 3344f18cd317a118c28482f97303600a2fe2ada6c79Al Viro size_t copied; 3354f18cd317a118c28482f97303600a2fe2ada6c79Al Viro 3364f18cd317a118c28482f97303600a2fe2ada6c79Al Viro kaddr = kmap_atomic(page); 3374f18cd317a118c28482f97303600a2fe2ada6c79Al Viro if (likely(i->nr_segs == 1)) { 3384f18cd317a118c28482f97303600a2fe2ada6c79Al Viro int left; 3394f18cd317a118c28482f97303600a2fe2ada6c79Al Viro char __user *buf = i->iov->iov_base + i->iov_offset; 3404f18cd317a118c28482f97303600a2fe2ada6c79Al Viro left = __copy_from_user_inatomic(kaddr + offset, buf, bytes); 3414f18cd317a118c28482f97303600a2fe2ada6c79Al Viro copied = bytes - left; 3424f18cd317a118c28482f97303600a2fe2ada6c79Al Viro } else { 3434f18cd317a118c28482f97303600a2fe2ada6c79Al Viro copied = __iovec_copy_from_user_inatomic(kaddr + offset, 3444f18cd317a118c28482f97303600a2fe2ada6c79Al Viro i->iov, i->iov_offset, bytes); 3454f18cd317a118c28482f97303600a2fe2ada6c79Al Viro } 3464f18cd317a118c28482f97303600a2fe2ada6c79Al Viro kunmap_atomic(kaddr); 3474f18cd317a118c28482f97303600a2fe2ada6c79Al Viro 3484f18cd317a118c28482f97303600a2fe2ada6c79Al Viro return copied; 3494f18cd317a118c28482f97303600a2fe2ada6c79Al Viro} 3504f18cd317a118c28482f97303600a2fe2ada6c79Al Viro 35162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virostatic void advance_iovec(struct iov_iter *i, size_t bytes) 3524f18cd317a118c28482f97303600a2fe2ada6c79Al Viro{ 3534f18cd317a118c28482f97303600a2fe2ada6c79Al Viro BUG_ON(i->count < bytes); 3544f18cd317a118c28482f97303600a2fe2ada6c79Al Viro 3554f18cd317a118c28482f97303600a2fe2ada6c79Al Viro if (likely(i->nr_segs == 1)) { 3564f18cd317a118c28482f97303600a2fe2ada6c79Al Viro i->iov_offset += bytes; 3574f18cd317a118c28482f97303600a2fe2ada6c79Al Viro i->count -= bytes; 3584f18cd317a118c28482f97303600a2fe2ada6c79Al Viro } else { 3594f18cd317a118c28482f97303600a2fe2ada6c79Al Viro const struct iovec *iov = i->iov; 3604f18cd317a118c28482f97303600a2fe2ada6c79Al Viro size_t base = i->iov_offset; 3614f18cd317a118c28482f97303600a2fe2ada6c79Al Viro unsigned long nr_segs = i->nr_segs; 3624f18cd317a118c28482f97303600a2fe2ada6c79Al Viro 3634f18cd317a118c28482f97303600a2fe2ada6c79Al Viro /* 3644f18cd317a118c28482f97303600a2fe2ada6c79Al Viro * The !iov->iov_len check ensures we skip over unlikely 3654f18cd317a118c28482f97303600a2fe2ada6c79Al Viro * zero-length segments (without overruning the iovec). 3664f18cd317a118c28482f97303600a2fe2ada6c79Al Viro */ 3674f18cd317a118c28482f97303600a2fe2ada6c79Al Viro while (bytes || unlikely(i->count && !iov->iov_len)) { 3684f18cd317a118c28482f97303600a2fe2ada6c79Al Viro int copy; 3694f18cd317a118c28482f97303600a2fe2ada6c79Al Viro 3704f18cd317a118c28482f97303600a2fe2ada6c79Al Viro copy = min(bytes, iov->iov_len - base); 3714f18cd317a118c28482f97303600a2fe2ada6c79Al Viro BUG_ON(!i->count || i->count < copy); 3724f18cd317a118c28482f97303600a2fe2ada6c79Al Viro i->count -= copy; 3734f18cd317a118c28482f97303600a2fe2ada6c79Al Viro bytes -= copy; 3744f18cd317a118c28482f97303600a2fe2ada6c79Al Viro base += copy; 3754f18cd317a118c28482f97303600a2fe2ada6c79Al Viro if (iov->iov_len == base) { 3764f18cd317a118c28482f97303600a2fe2ada6c79Al Viro iov++; 3774f18cd317a118c28482f97303600a2fe2ada6c79Al Viro nr_segs--; 3784f18cd317a118c28482f97303600a2fe2ada6c79Al Viro base = 0; 3794f18cd317a118c28482f97303600a2fe2ada6c79Al Viro } 3804f18cd317a118c28482f97303600a2fe2ada6c79Al Viro } 3814f18cd317a118c28482f97303600a2fe2ada6c79Al Viro i->iov = iov; 3824f18cd317a118c28482f97303600a2fe2ada6c79Al Viro i->iov_offset = base; 3834f18cd317a118c28482f97303600a2fe2ada6c79Al Viro i->nr_segs = nr_segs; 3844f18cd317a118c28482f97303600a2fe2ada6c79Al Viro } 3854f18cd317a118c28482f97303600a2fe2ada6c79Al Viro} 3864f18cd317a118c28482f97303600a2fe2ada6c79Al Viro 3874f18cd317a118c28482f97303600a2fe2ada6c79Al Viro/* 3884f18cd317a118c28482f97303600a2fe2ada6c79Al Viro * Fault in the first iovec of the given iov_iter, to a maximum length 3894f18cd317a118c28482f97303600a2fe2ada6c79Al Viro * of bytes. Returns 0 on success, or non-zero if the memory could not be 3904f18cd317a118c28482f97303600a2fe2ada6c79Al Viro * accessed (ie. because it is an invalid address). 3914f18cd317a118c28482f97303600a2fe2ada6c79Al Viro * 3924f18cd317a118c28482f97303600a2fe2ada6c79Al Viro * writev-intensive code may want this to prefault several iovecs -- that 3934f18cd317a118c28482f97303600a2fe2ada6c79Al Viro * would be possible (callers must not rely on the fact that _only_ the 3944f18cd317a118c28482f97303600a2fe2ada6c79Al Viro * first iovec will be faulted with the current implementation). 3954f18cd317a118c28482f97303600a2fe2ada6c79Al Viro */ 3964f18cd317a118c28482f97303600a2fe2ada6c79Al Viroint iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes) 3974f18cd317a118c28482f97303600a2fe2ada6c79Al Viro{ 39862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (!(i->type & ITER_BVEC)) { 39962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro char __user *buf = i->iov->iov_base + i->iov_offset; 40062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro bytes = min(bytes, i->iov->iov_len - i->iov_offset); 40162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return fault_in_pages_readable(buf, bytes); 40262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro } 40362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return 0; 4044f18cd317a118c28482f97303600a2fe2ada6c79Al Viro} 4054f18cd317a118c28482f97303600a2fe2ada6c79Al ViroEXPORT_SYMBOL(iov_iter_fault_in_readable); 4064f18cd317a118c28482f97303600a2fe2ada6c79Al Viro 40762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virostatic unsigned long alignment_iovec(const struct iov_iter *i) 408886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro{ 409886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro const struct iovec *iov = i->iov; 410886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro unsigned long res; 411886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro size_t size = i->count; 412886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro size_t n; 413886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro 414886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro if (!size) 415886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro return 0; 416886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro 417886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro res = (unsigned long)iov->iov_base + i->iov_offset; 418886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro n = iov->iov_len - i->iov_offset; 419886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro if (n >= size) 420886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro return res | size; 421886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro size -= n; 422886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro res |= n; 423886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro while (size > (++iov)->iov_len) { 424886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro res |= (unsigned long)iov->iov_base | iov->iov_len; 425886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro size -= iov->iov_len; 426886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro } 427886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro res |= (unsigned long)iov->iov_base | size; 428886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro return res; 429886a39115005ced8b15ab067c9c2a8d546b40a5eAl Viro} 43071d8e532b1549a478e6a6a8a44f309d050294d00Al Viro 43171d8e532b1549a478e6a6a8a44f309d050294d00Al Virovoid iov_iter_init(struct iov_iter *i, int direction, 43271d8e532b1549a478e6a6a8a44f309d050294d00Al Viro const struct iovec *iov, unsigned long nr_segs, 43371d8e532b1549a478e6a6a8a44f309d050294d00Al Viro size_t count) 43471d8e532b1549a478e6a6a8a44f309d050294d00Al Viro{ 43571d8e532b1549a478e6a6a8a44f309d050294d00Al Viro /* It will get better. Eventually... */ 43671d8e532b1549a478e6a6a8a44f309d050294d00Al Viro if (segment_eq(get_fs(), KERNEL_DS)) 43762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro direction |= ITER_KVEC; 43871d8e532b1549a478e6a6a8a44f309d050294d00Al Viro i->type = direction; 43971d8e532b1549a478e6a6a8a44f309d050294d00Al Viro i->iov = iov; 44071d8e532b1549a478e6a6a8a44f309d050294d00Al Viro i->nr_segs = nr_segs; 44171d8e532b1549a478e6a6a8a44f309d050294d00Al Viro i->iov_offset = 0; 44271d8e532b1549a478e6a6a8a44f309d050294d00Al Viro i->count = count; 44371d8e532b1549a478e6a6a8a44f309d050294d00Al Viro} 44471d8e532b1549a478e6a6a8a44f309d050294d00Al ViroEXPORT_SYMBOL(iov_iter_init); 4457b2c99d15559e285384c742db52316802e24b0bdAl Viro 44662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virostatic ssize_t get_pages_iovec(struct iov_iter *i, 4472c80929c4c4d54e568b07ab85877d5fd38f4b02fMiklos Szeredi struct page **pages, size_t maxsize, unsigned maxpages, 4487b2c99d15559e285384c742db52316802e24b0bdAl Viro size_t *start) 4497b2c99d15559e285384c742db52316802e24b0bdAl Viro{ 4507b2c99d15559e285384c742db52316802e24b0bdAl Viro size_t offset = i->iov_offset; 4517b2c99d15559e285384c742db52316802e24b0bdAl Viro const struct iovec *iov = i->iov; 4527b2c99d15559e285384c742db52316802e24b0bdAl Viro size_t len; 4537b2c99d15559e285384c742db52316802e24b0bdAl Viro unsigned long addr; 4547b2c99d15559e285384c742db52316802e24b0bdAl Viro int n; 4557b2c99d15559e285384c742db52316802e24b0bdAl Viro int res; 4567b2c99d15559e285384c742db52316802e24b0bdAl Viro 4577b2c99d15559e285384c742db52316802e24b0bdAl Viro len = iov->iov_len - offset; 4587b2c99d15559e285384c742db52316802e24b0bdAl Viro if (len > i->count) 4597b2c99d15559e285384c742db52316802e24b0bdAl Viro len = i->count; 4602c80929c4c4d54e568b07ab85877d5fd38f4b02fMiklos Szeredi if (len > maxsize) 4612c80929c4c4d54e568b07ab85877d5fd38f4b02fMiklos Szeredi len = maxsize; 4627b2c99d15559e285384c742db52316802e24b0bdAl Viro addr = (unsigned long)iov->iov_base + offset; 4637b2c99d15559e285384c742db52316802e24b0bdAl Viro len += *start = addr & (PAGE_SIZE - 1); 464c7f3888ad7f0932a87fb76e6e4edff2a90cc7920Al Viro if (len > maxpages * PAGE_SIZE) 465c7f3888ad7f0932a87fb76e6e4edff2a90cc7920Al Viro len = maxpages * PAGE_SIZE; 4667b2c99d15559e285384c742db52316802e24b0bdAl Viro addr &= ~(PAGE_SIZE - 1); 4677b2c99d15559e285384c742db52316802e24b0bdAl Viro n = (len + PAGE_SIZE - 1) / PAGE_SIZE; 4687b2c99d15559e285384c742db52316802e24b0bdAl Viro res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages); 4697b2c99d15559e285384c742db52316802e24b0bdAl Viro if (unlikely(res < 0)) 4707b2c99d15559e285384c742db52316802e24b0bdAl Viro return res; 4717b2c99d15559e285384c742db52316802e24b0bdAl Viro return (res == n ? len : res * PAGE_SIZE) - *start; 4727b2c99d15559e285384c742db52316802e24b0bdAl Viro} 473f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro 47462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virostatic ssize_t get_pages_alloc_iovec(struct iov_iter *i, 47591f79c43d1b54d7154b118860d81b39bad07dfffAl Viro struct page ***pages, size_t maxsize, 47691f79c43d1b54d7154b118860d81b39bad07dfffAl Viro size_t *start) 47791f79c43d1b54d7154b118860d81b39bad07dfffAl Viro{ 47891f79c43d1b54d7154b118860d81b39bad07dfffAl Viro size_t offset = i->iov_offset; 47991f79c43d1b54d7154b118860d81b39bad07dfffAl Viro const struct iovec *iov = i->iov; 48091f79c43d1b54d7154b118860d81b39bad07dfffAl Viro size_t len; 48191f79c43d1b54d7154b118860d81b39bad07dfffAl Viro unsigned long addr; 48291f79c43d1b54d7154b118860d81b39bad07dfffAl Viro void *p; 48391f79c43d1b54d7154b118860d81b39bad07dfffAl Viro int n; 48491f79c43d1b54d7154b118860d81b39bad07dfffAl Viro int res; 48591f79c43d1b54d7154b118860d81b39bad07dfffAl Viro 48691f79c43d1b54d7154b118860d81b39bad07dfffAl Viro len = iov->iov_len - offset; 48791f79c43d1b54d7154b118860d81b39bad07dfffAl Viro if (len > i->count) 48891f79c43d1b54d7154b118860d81b39bad07dfffAl Viro len = i->count; 48991f79c43d1b54d7154b118860d81b39bad07dfffAl Viro if (len > maxsize) 49091f79c43d1b54d7154b118860d81b39bad07dfffAl Viro len = maxsize; 49191f79c43d1b54d7154b118860d81b39bad07dfffAl Viro addr = (unsigned long)iov->iov_base + offset; 49291f79c43d1b54d7154b118860d81b39bad07dfffAl Viro len += *start = addr & (PAGE_SIZE - 1); 49391f79c43d1b54d7154b118860d81b39bad07dfffAl Viro addr &= ~(PAGE_SIZE - 1); 49491f79c43d1b54d7154b118860d81b39bad07dfffAl Viro n = (len + PAGE_SIZE - 1) / PAGE_SIZE; 49591f79c43d1b54d7154b118860d81b39bad07dfffAl Viro 49691f79c43d1b54d7154b118860d81b39bad07dfffAl Viro p = kmalloc(n * sizeof(struct page *), GFP_KERNEL); 49791f79c43d1b54d7154b118860d81b39bad07dfffAl Viro if (!p) 49891f79c43d1b54d7154b118860d81b39bad07dfffAl Viro p = vmalloc(n * sizeof(struct page *)); 49991f79c43d1b54d7154b118860d81b39bad07dfffAl Viro if (!p) 50091f79c43d1b54d7154b118860d81b39bad07dfffAl Viro return -ENOMEM; 50191f79c43d1b54d7154b118860d81b39bad07dfffAl Viro 50291f79c43d1b54d7154b118860d81b39bad07dfffAl Viro res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p); 50391f79c43d1b54d7154b118860d81b39bad07dfffAl Viro if (unlikely(res < 0)) { 50491f79c43d1b54d7154b118860d81b39bad07dfffAl Viro kvfree(p); 50591f79c43d1b54d7154b118860d81b39bad07dfffAl Viro return res; 50691f79c43d1b54d7154b118860d81b39bad07dfffAl Viro } 50791f79c43d1b54d7154b118860d81b39bad07dfffAl Viro *pages = p; 50891f79c43d1b54d7154b118860d81b39bad07dfffAl Viro return (res == n ? len : res * PAGE_SIZE) - *start; 50991f79c43d1b54d7154b118860d81b39bad07dfffAl Viro} 51091f79c43d1b54d7154b118860d81b39bad07dfffAl Viro 51162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virostatic int iov_iter_npages_iovec(const struct iov_iter *i, int maxpages) 512f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro{ 513f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro size_t offset = i->iov_offset; 514f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro size_t size = i->count; 515f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro const struct iovec *iov = i->iov; 516f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro int npages = 0; 517f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro int n; 518f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro 519f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro for (n = 0; size && n < i->nr_segs; n++, iov++) { 520f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro unsigned long addr = (unsigned long)iov->iov_base + offset; 521f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro size_t len = iov->iov_len - offset; 522f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro offset = 0; 523f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro if (unlikely(!len)) /* empty segment */ 524f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro continue; 525f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro if (len > size) 526f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro len = size; 527f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro npages += (addr + len + PAGE_SIZE - 1) / PAGE_SIZE 528f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro - addr / PAGE_SIZE; 529f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro if (npages >= maxpages) /* don't bother going further */ 530f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro return maxpages; 531f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro size -= len; 532f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro offset = 0; 533f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro } 534f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro return min(npages, maxpages); 535f67da30c1d5fc9e341bc8121708874bfd7b31e45Al Viro} 53662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 53762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virostatic void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len) 53862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro{ 53962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro char *from = kmap_atomic(page); 54062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro memcpy(to, from + offset, len); 54162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro kunmap_atomic(from); 54262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro} 54362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 54462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virostatic void memcpy_to_page(struct page *page, size_t offset, char *from, size_t len) 54562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro{ 54662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro char *to = kmap_atomic(page); 54762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro memcpy(to + offset, from, len); 54862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro kunmap_atomic(to); 54962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro} 55062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 551c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcoxstatic void memzero_page(struct page *page, size_t offset, size_t len) 552c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox{ 553c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox char *addr = kmap_atomic(page); 554c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox memset(addr + offset, 0, len); 555c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox kunmap_atomic(addr); 556c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox} 557c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 558c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcoxstatic size_t copy_to_iter_bvec(void *from, size_t bytes, struct iov_iter *i) 55962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro{ 56062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro size_t skip, copy, wanted; 56162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro const struct bio_vec *bvec; 56262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 56362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (unlikely(bytes > i->count)) 56462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro bytes = i->count; 56562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 56662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (unlikely(!bytes)) 56762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return 0; 56862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 56962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro wanted = bytes; 57062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro bvec = i->bvec; 57162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro skip = i->iov_offset; 57262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro copy = min_t(size_t, bytes, bvec->bv_len - skip); 57362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 57462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro memcpy_to_page(bvec->bv_page, skip + bvec->bv_offset, from, copy); 57562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro skip += copy; 57662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro from += copy; 57762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro bytes -= copy; 57862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro while (bytes) { 57962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro bvec++; 58062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro copy = min(bytes, (size_t)bvec->bv_len); 58162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro memcpy_to_page(bvec->bv_page, bvec->bv_offset, from, copy); 58262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro skip = copy; 58362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro from += copy; 58462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro bytes -= copy; 58562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro } 58662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (skip == bvec->bv_len) { 58762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro bvec++; 58862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro skip = 0; 58962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro } 59062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro i->count -= wanted - bytes; 59162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro i->nr_segs -= bvec - i->bvec; 59262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro i->bvec = bvec; 59362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro i->iov_offset = skip; 59462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return wanted - bytes; 59562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro} 59662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 597c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcoxstatic size_t copy_from_iter_bvec(void *to, size_t bytes, struct iov_iter *i) 59862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro{ 59962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro size_t skip, copy, wanted; 60062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro const struct bio_vec *bvec; 60162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 60262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (unlikely(bytes > i->count)) 60362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro bytes = i->count; 60462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 60562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (unlikely(!bytes)) 60662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return 0; 60762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 60862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro wanted = bytes; 60962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro bvec = i->bvec; 61062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro skip = i->iov_offset; 61162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 61262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro copy = min(bytes, bvec->bv_len - skip); 61362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 61462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro memcpy_from_page(to, bvec->bv_page, bvec->bv_offset + skip, copy); 61562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 61662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro to += copy; 61762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro skip += copy; 61862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro bytes -= copy; 61962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 62062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro while (bytes) { 62162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro bvec++; 62262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro copy = min(bytes, (size_t)bvec->bv_len); 62362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro memcpy_from_page(to, bvec->bv_page, bvec->bv_offset, copy); 62462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro skip = copy; 62562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro to += copy; 62662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro bytes -= copy; 62762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro } 62862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (skip == bvec->bv_len) { 62962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro bvec++; 63062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro skip = 0; 63162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro } 63262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro i->count -= wanted; 63362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro i->nr_segs -= bvec - i->bvec; 63462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro i->bvec = bvec; 63562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro i->iov_offset = skip; 63662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return wanted; 63762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro} 63862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 639c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcoxstatic size_t copy_page_to_iter_bvec(struct page *page, size_t offset, 640c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox size_t bytes, struct iov_iter *i) 641c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox{ 642c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox void *kaddr = kmap_atomic(page); 643c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox size_t wanted = copy_to_iter_bvec(kaddr + offset, bytes, i); 644c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox kunmap_atomic(kaddr); 645c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox return wanted; 646c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox} 647c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 648c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcoxstatic size_t copy_page_from_iter_bvec(struct page *page, size_t offset, 649c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox size_t bytes, struct iov_iter *i) 650c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox{ 651c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox void *kaddr = kmap_atomic(page); 652c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox size_t wanted = copy_from_iter_bvec(kaddr + offset, bytes, i); 653c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox kunmap_atomic(kaddr); 654c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox return wanted; 655c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox} 656c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 657c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcoxstatic size_t zero_bvec(size_t bytes, struct iov_iter *i) 658c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox{ 659c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox size_t skip, copy, wanted; 660c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox const struct bio_vec *bvec; 661c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 662c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox if (unlikely(bytes > i->count)) 663c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox bytes = i->count; 664c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 665c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox if (unlikely(!bytes)) 666c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox return 0; 667c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 668c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox wanted = bytes; 669c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox bvec = i->bvec; 670c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox skip = i->iov_offset; 671c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox copy = min_t(size_t, bytes, bvec->bv_len - skip); 672c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 673c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox memzero_page(bvec->bv_page, skip + bvec->bv_offset, copy); 674c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox skip += copy; 675c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox bytes -= copy; 676c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox while (bytes) { 677c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox bvec++; 678c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox copy = min(bytes, (size_t)bvec->bv_len); 679c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox memzero_page(bvec->bv_page, bvec->bv_offset, copy); 680c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox skip = copy; 681c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox bytes -= copy; 682c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox } 683c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox if (skip == bvec->bv_len) { 684c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox bvec++; 685c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox skip = 0; 686c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox } 687c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox i->count -= wanted - bytes; 688c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox i->nr_segs -= bvec - i->bvec; 689c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox i->bvec = bvec; 690c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox i->iov_offset = skip; 691c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox return wanted - bytes; 692c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox} 693c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 69462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virostatic size_t copy_from_user_bvec(struct page *page, 69562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro struct iov_iter *i, unsigned long offset, size_t bytes) 69662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro{ 69762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro char *kaddr; 69862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro size_t left; 69962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro const struct bio_vec *bvec; 70062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro size_t base = i->iov_offset; 70162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 70262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro kaddr = kmap_atomic(page); 70362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro for (left = bytes, bvec = i->bvec; left; bvec++, base = 0) { 70462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro size_t copy = min(left, bvec->bv_len - base); 70562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (!bvec->bv_len) 70662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro continue; 70762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro memcpy_from_page(kaddr + offset, bvec->bv_page, 70862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro bvec->bv_offset + base, copy); 70962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro offset += copy; 71062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro left -= copy; 71162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro } 71262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro kunmap_atomic(kaddr); 71362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return bytes; 71462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro} 71562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 71662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virostatic void advance_bvec(struct iov_iter *i, size_t bytes) 71762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro{ 71862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro BUG_ON(i->count < bytes); 71962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 72062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (likely(i->nr_segs == 1)) { 72162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro i->iov_offset += bytes; 72262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro i->count -= bytes; 72362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro } else { 72462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro const struct bio_vec *bvec = i->bvec; 72562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro size_t base = i->iov_offset; 72662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro unsigned long nr_segs = i->nr_segs; 72762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 72862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro /* 72962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro * The !iov->iov_len check ensures we skip over unlikely 73062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro * zero-length segments (without overruning the iovec). 73162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro */ 73262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro while (bytes || unlikely(i->count && !bvec->bv_len)) { 73362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro int copy; 73462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 73562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro copy = min(bytes, bvec->bv_len - base); 73662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro BUG_ON(!i->count || i->count < copy); 73762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro i->count -= copy; 73862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro bytes -= copy; 73962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro base += copy; 74062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (bvec->bv_len == base) { 74162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro bvec++; 74262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro nr_segs--; 74362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro base = 0; 74462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro } 74562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro } 74662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro i->bvec = bvec; 74762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro i->iov_offset = base; 74862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro i->nr_segs = nr_segs; 74962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro } 75062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro} 75162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 75262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virostatic unsigned long alignment_bvec(const struct iov_iter *i) 75362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro{ 75462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro const struct bio_vec *bvec = i->bvec; 75562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro unsigned long res; 75662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro size_t size = i->count; 75762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro size_t n; 75862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 75962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (!size) 76062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return 0; 76162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 76262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro res = bvec->bv_offset + i->iov_offset; 76362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro n = bvec->bv_len - i->iov_offset; 76462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (n >= size) 76562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return res | size; 76662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro size -= n; 76762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro res |= n; 76862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro while (size > (++bvec)->bv_len) { 76962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro res |= bvec->bv_offset | bvec->bv_len; 77062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro size -= bvec->bv_len; 77162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro } 77262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro res |= bvec->bv_offset | size; 77362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return res; 77462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro} 77562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 77662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virostatic ssize_t get_pages_bvec(struct iov_iter *i, 7772c80929c4c4d54e568b07ab85877d5fd38f4b02fMiklos Szeredi struct page **pages, size_t maxsize, unsigned maxpages, 77862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro size_t *start) 77962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro{ 78062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro const struct bio_vec *bvec = i->bvec; 78162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro size_t len = bvec->bv_len - i->iov_offset; 78262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (len > i->count) 78362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro len = i->count; 7842c80929c4c4d54e568b07ab85877d5fd38f4b02fMiklos Szeredi if (len > maxsize) 7852c80929c4c4d54e568b07ab85877d5fd38f4b02fMiklos Szeredi len = maxsize; 786c7f3888ad7f0932a87fb76e6e4edff2a90cc7920Al Viro /* can't be more than PAGE_SIZE */ 78762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro *start = bvec->bv_offset + i->iov_offset; 78862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 78962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro get_page(*pages = bvec->bv_page); 79062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 79162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return len; 79262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro} 79362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 79462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virostatic ssize_t get_pages_alloc_bvec(struct iov_iter *i, 79562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro struct page ***pages, size_t maxsize, 79662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro size_t *start) 79762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro{ 79862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro const struct bio_vec *bvec = i->bvec; 79962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro size_t len = bvec->bv_len - i->iov_offset; 80062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (len > i->count) 80162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro len = i->count; 80262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (len > maxsize) 80362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro len = maxsize; 80462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro *start = bvec->bv_offset + i->iov_offset; 80562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 80662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro *pages = kmalloc(sizeof(struct page *), GFP_KERNEL); 80762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (!*pages) 80862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return -ENOMEM; 80962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 81062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro get_page(**pages = bvec->bv_page); 81162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 81262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return len; 81362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro} 81462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 81562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virostatic int iov_iter_npages_bvec(const struct iov_iter *i, int maxpages) 81662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro{ 81762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro size_t offset = i->iov_offset; 81862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro size_t size = i->count; 81962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro const struct bio_vec *bvec = i->bvec; 82062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro int npages = 0; 82162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro int n; 82262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 82362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro for (n = 0; size && n < i->nr_segs; n++, bvec++) { 82462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro size_t len = bvec->bv_len - offset; 82562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro offset = 0; 82662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (unlikely(!len)) /* empty segment */ 82762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro continue; 82862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (len > size) 82962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro len = size; 83062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro npages++; 83162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (npages >= maxpages) /* don't bother going further */ 83262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return maxpages; 83362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro size -= len; 83462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro offset = 0; 83562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro } 83662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return min(npages, maxpages); 83762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro} 83862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 83962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virosize_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, 84062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro struct iov_iter *i) 84162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro{ 84262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (i->type & ITER_BVEC) 84362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return copy_page_to_iter_bvec(page, offset, bytes, i); 84462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro else 84562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return copy_page_to_iter_iovec(page, offset, bytes, i); 84662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro} 84762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al ViroEXPORT_SYMBOL(copy_page_to_iter); 84862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 84962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virosize_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, 85062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro struct iov_iter *i) 85162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro{ 85262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (i->type & ITER_BVEC) 85362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return copy_page_from_iter_bvec(page, offset, bytes, i); 85462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro else 85562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return copy_page_from_iter_iovec(page, offset, bytes, i); 85662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro} 85762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al ViroEXPORT_SYMBOL(copy_page_from_iter); 85862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 859c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcoxsize_t copy_to_iter(void *addr, size_t bytes, struct iov_iter *i) 860c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox{ 861c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox if (i->type & ITER_BVEC) 862c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox return copy_to_iter_bvec(addr, bytes, i); 863c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox else 864c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox return copy_to_iter_iovec(addr, bytes, i); 865c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox} 866c35e02480014f7a86e264a2fda39a568690163daMatthew WilcoxEXPORT_SYMBOL(copy_to_iter); 867c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 868c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcoxsize_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) 869c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox{ 870c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox if (i->type & ITER_BVEC) 871c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox return copy_from_iter_bvec(addr, bytes, i); 872c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox else 873c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox return copy_from_iter_iovec(addr, bytes, i); 874c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox} 875c35e02480014f7a86e264a2fda39a568690163daMatthew WilcoxEXPORT_SYMBOL(copy_from_iter); 876c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 877c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcoxsize_t iov_iter_zero(size_t bytes, struct iov_iter *i) 878c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox{ 879c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox if (i->type & ITER_BVEC) { 880c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox return zero_bvec(bytes, i); 881c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox } else { 882c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox return zero_iovec(bytes, i); 883c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox } 884c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox} 885c35e02480014f7a86e264a2fda39a568690163daMatthew WilcoxEXPORT_SYMBOL(iov_iter_zero); 886c35e02480014f7a86e264a2fda39a568690163daMatthew Wilcox 88762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virosize_t iov_iter_copy_from_user_atomic(struct page *page, 88862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro struct iov_iter *i, unsigned long offset, size_t bytes) 88962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro{ 89062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (i->type & ITER_BVEC) 89162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return copy_from_user_bvec(page, i, offset, bytes); 89262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro else 89362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return copy_from_user_atomic_iovec(page, i, offset, bytes); 89462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro} 89562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al ViroEXPORT_SYMBOL(iov_iter_copy_from_user_atomic); 89662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 89762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virovoid iov_iter_advance(struct iov_iter *i, size_t size) 89862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro{ 89962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (i->type & ITER_BVEC) 90062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro advance_bvec(i, size); 90162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro else 90262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro advance_iovec(i, size); 90362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro} 90462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al ViroEXPORT_SYMBOL(iov_iter_advance); 90562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 90662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro/* 90762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro * Return the count of just the current iov_iter segment. 90862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro */ 90962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virosize_t iov_iter_single_seg_count(const struct iov_iter *i) 91062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro{ 91162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (i->nr_segs == 1) 91262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return i->count; 91362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro else if (i->type & ITER_BVEC) 91462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return min(i->count, i->bvec->bv_len - i->iov_offset); 915ad0eab9293485d1c06237e9249f6d4dfa3d93d4dPaul Mackerras else 916ad0eab9293485d1c06237e9249f6d4dfa3d93d4dPaul Mackerras return min(i->count, i->iov->iov_len - i->iov_offset); 91762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro} 91862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al ViroEXPORT_SYMBOL(iov_iter_single_seg_count); 91962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 92062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virounsigned long iov_iter_alignment(const struct iov_iter *i) 92162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro{ 92262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (i->type & ITER_BVEC) 92362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return alignment_bvec(i); 92462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro else 92562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return alignment_iovec(i); 92662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro} 92762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al ViroEXPORT_SYMBOL(iov_iter_alignment); 92862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 92962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virossize_t iov_iter_get_pages(struct iov_iter *i, 9302c80929c4c4d54e568b07ab85877d5fd38f4b02fMiklos Szeredi struct page **pages, size_t maxsize, unsigned maxpages, 93162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro size_t *start) 93262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro{ 93362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (i->type & ITER_BVEC) 9342c80929c4c4d54e568b07ab85877d5fd38f4b02fMiklos Szeredi return get_pages_bvec(i, pages, maxsize, maxpages, start); 93562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro else 9362c80929c4c4d54e568b07ab85877d5fd38f4b02fMiklos Szeredi return get_pages_iovec(i, pages, maxsize, maxpages, start); 93762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro} 93862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al ViroEXPORT_SYMBOL(iov_iter_get_pages); 93962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 94062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Virossize_t iov_iter_get_pages_alloc(struct iov_iter *i, 94162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro struct page ***pages, size_t maxsize, 94262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro size_t *start) 94362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro{ 94462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (i->type & ITER_BVEC) 94562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return get_pages_alloc_bvec(i, pages, maxsize, start); 94662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro else 94762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return get_pages_alloc_iovec(i, pages, maxsize, start); 94862a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro} 94962a8067a7f35dba2de501c9cb00e4cf36da90bc0Al ViroEXPORT_SYMBOL(iov_iter_get_pages_alloc); 95062a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro 95162a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viroint iov_iter_npages(const struct iov_iter *i, int maxpages) 95262a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro{ 95362a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro if (i->type & ITER_BVEC) 95462a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return iov_iter_npages_bvec(i, maxpages); 95562a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro else 95662a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro return iov_iter_npages_iovec(i, maxpages); 95762a8067a7f35dba2de501c9cb00e4cf36da90bc0Al Viro} 958f67da30c1d5fc9e341bc8121708874bfd7b31e45Al ViroEXPORT_SYMBOL(iov_iter_npages); 959