dm-io.c revision f00b16ad665a9b489d46f612679181f3f914917b
11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (C) 2003 Sistina Software 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This file is released under the GPL. 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include "dm-io.h" 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/bio.h> 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mempool.h> 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/module.h> 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/sched.h> 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/slab.h> 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic struct bio_set *_bios; 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* FIXME: can we shrink this ? */ 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstruct io { 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long error; 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_t count; 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct task_struct *sleeper; 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io_notify_fn callback; 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *context; 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}; 251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * io contexts are only dynamically allocated for asynchronous 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * io. Since async io is likely to be the majority of io we'll 291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * have the same number of io contexts as buffer heads ! (FIXME: 301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * must reduce this). 311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic unsigned _num_ios; 331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic mempool_t *_io_pool; 341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic unsigned int pages_to_ios(unsigned int pages) 361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 4 * pages; /* too many ? */ 381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic int resize_pool(unsigned int new_ios) 411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int r = 0; 431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (_io_pool) { 451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (new_ios == 0) { 461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* free off the pool */ 471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mempool_destroy(_io_pool); 481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds _io_pool = NULL; 491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bioset_free(_bios); 501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } else { 521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* resize the pool */ 531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds r = mempool_resize(_io_pool, new_ios, GFP_KERNEL); 541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } else { 571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* create new pool */ 580eaae62abaa1ad1f231932b6cdd9fb1b91df6651Matthew Dobson _io_pool = mempool_create_kmalloc_pool(new_ios, 590eaae62abaa1ad1f231932b6cdd9fb1b91df6651Matthew Dobson sizeof(struct io)); 601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!_io_pool) 611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -ENOMEM; 621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds _bios = bioset_create(16, 16, 4); 641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!_bios) { 651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mempool_destroy(_io_pool); 661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds _io_pool = NULL; 671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -ENOMEM; 681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!r) 721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds _num_ios = new_ios; 731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return r; 751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint dm_io_get(unsigned int num_pages) 781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return resize_pool(_num_ios + pages_to_ios(num_pages)); 801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid dm_io_put(unsigned int num_pages) 831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds resize_pool(_num_ios - pages_to_ios(num_pages)); 851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*----------------------------------------------------------------- 881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * We need to keep track of which region a bio is doing io for. 891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * In order to save a memory allocation we store this the last 901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * bvec which we know is unused (blech). 911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * XXX This is ugly and can OOPS with some configs... find another way. 921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *---------------------------------------------------------------*/ 931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic inline void bio_set_region(struct bio *bio, unsigned region) 941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 95f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen bio->bi_io_vec[bio->bi_max_vecs].bv_len = region; 961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic inline unsigned bio_get_region(struct bio *bio) 991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 100f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen return bio->bi_io_vec[bio->bi_max_vecs].bv_len; 1011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*----------------------------------------------------------------- 1041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * We need an io object to keep track of the number of bios that 1051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * have been dispatched for a particular io. 1061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *---------------------------------------------------------------*/ 1071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void dec_count(struct io *io, unsigned int region, int error) 1081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (error) 1101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds set_bit(region, &io->error); 1111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (atomic_dec_and_test(&io->count)) { 1131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (io->sleeper) 1141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds wake_up_process(io->sleeper); 1151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds else { 1171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int r = io->error; 1181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io_notify_fn fn = io->callback; 1191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *context = io->context; 1201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mempool_free(io, _io_pool); 1221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fn(r, context); 1231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic int endio(struct bio *bio, unsigned int done, int error) 1281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct io *io = (struct io *) bio->bi_private; 1301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* keep going until we've finished */ 1321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (bio->bi_size) 1331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 1; 1341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (error && bio_data_dir(bio) == READ) 1361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds zero_fill_bio(bio); 1371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dec_count(io, bio_get_region(bio), error); 139f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen bio->bi_max_vecs++; 1401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio_put(bio); 1411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 1431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*----------------------------------------------------------------- 1461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * These little objects provide an abstraction for getting a new 1471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * destination page for io. 1481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *---------------------------------------------------------------*/ 1491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstruct dpages { 1501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void (*get_page)(struct dpages *dp, 1511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page **p, unsigned long *len, unsigned *offset); 1521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void (*next_page)(struct dpages *dp); 1531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned context_u; 1551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *context_ptr; 1561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}; 1571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 1591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Functions for getting the pages from a list. 1601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 1611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void list_get_page(struct dpages *dp, 1621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page **p, unsigned long *len, unsigned *offset) 1631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned o = dp->context_u; 1651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page_list *pl = (struct page_list *) dp->context_ptr; 1661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *p = pl->page; 1681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *len = PAGE_SIZE - o; 1691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *offset = o; 1701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void list_next_page(struct dpages *dp) 1731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page_list *pl = (struct page_list *) dp->context_ptr; 1751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr = pl->next; 1761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_u = 0; 1771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void list_dp_init(struct dpages *dp, struct page_list *pl, unsigned offset) 1801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->get_page = list_get_page; 1821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->next_page = list_next_page; 1831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_u = offset; 1841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr = pl; 1851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 1881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Functions for getting the pages from a bvec. 1891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 1901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void bvec_get_page(struct dpages *dp, 1911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page **p, unsigned long *len, unsigned *offset) 1921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct bio_vec *bvec = (struct bio_vec *) dp->context_ptr; 1941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *p = bvec->bv_page; 1951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *len = bvec->bv_len; 1961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *offset = bvec->bv_offset; 1971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void bvec_next_page(struct dpages *dp) 2001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct bio_vec *bvec = (struct bio_vec *) dp->context_ptr; 2021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr = bvec + 1; 2031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void bvec_dp_init(struct dpages *dp, struct bio_vec *bvec) 2061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->get_page = bvec_get_page; 2081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->next_page = bvec_next_page; 2091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr = bvec; 2101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void vm_get_page(struct dpages *dp, 2131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page **p, unsigned long *len, unsigned *offset) 2141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *p = vmalloc_to_page(dp->context_ptr); 2161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *offset = dp->context_u; 2171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *len = PAGE_SIZE - dp->context_u; 2181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void vm_next_page(struct dpages *dp) 2211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr += PAGE_SIZE - dp->context_u; 2231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_u = 0; 2241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void vm_dp_init(struct dpages *dp, void *data) 2271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->get_page = vm_get_page; 2291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->next_page = vm_next_page; 2301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_u = ((unsigned long) data) & (PAGE_SIZE - 1); 2311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr = data; 2321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2343676347a5e216a7fec7f8eedbbcf8bed6b9c4e40Peter Osterlundstatic void dm_bio_destructor(struct bio *bio) 2353676347a5e216a7fec7f8eedbbcf8bed6b9c4e40Peter Osterlund{ 2363676347a5e216a7fec7f8eedbbcf8bed6b9c4e40Peter Osterlund bio_free(bio, _bios); 2373676347a5e216a7fec7f8eedbbcf8bed6b9c4e40Peter Osterlund} 2383676347a5e216a7fec7f8eedbbcf8bed6b9c4e40Peter Osterlund 2391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*----------------------------------------------------------------- 2401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * IO routines that accept a list of pages. 2411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *---------------------------------------------------------------*/ 2421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void do_region(int rw, unsigned int region, struct io_region *where, 2431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct dpages *dp, struct io *io) 2441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct bio *bio; 2461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page *page; 2471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long len; 2481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned offset; 2491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned num_bvecs; 2501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds sector_t remaining = where->count; 2511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (remaining) { 2531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 254f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen * Allocate a suitably sized-bio: we add an extra 255f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen * bvec for bio_get/set_region() and decrement bi_max_vecs 256f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen * to hide it from bio_add_page(). 2571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 258f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen num_bvecs = (remaining / (PAGE_SIZE >> SECTOR_SHIFT)) + 2; 2591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio = bio_alloc_bioset(GFP_NOIO, num_bvecs, _bios); 2601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio->bi_sector = where->sector + (where->count - remaining); 2611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio->bi_bdev = where->bdev; 2621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio->bi_end_io = endio; 2631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio->bi_private = io; 2643676347a5e216a7fec7f8eedbbcf8bed6b9c4e40Peter Osterlund bio->bi_destructor = dm_bio_destructor; 265f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen bio->bi_max_vecs--; 2661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio_set_region(bio, region); 2671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 2691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Try and add as many pages as possible. 2701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 2711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (remaining) { 2721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->get_page(dp, &page, &len, &offset); 2731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds len = min(len, to_bytes(remaining)); 2741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!bio_add_page(bio, page, len, offset)) 2751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds break; 2761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds offset = 0; 2781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds remaining -= to_sector(len); 2791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->next_page(dp); 2801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_inc(&io->count); 2831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds submit_bio(rw, bio); 2841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void dispatch_io(int rw, unsigned int num_regions, 2881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct io_region *where, struct dpages *dp, 2891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct io *io, int sync) 2901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int i; 2921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct dpages old_pages = *dp; 2931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (sync) 2951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds rw |= (1 << BIO_RW_SYNC); 2961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 2981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * For multiple regions we need to be careful to rewind 2991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * the dp object for each call to do_region. 3001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 3011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds for (i = 0; i < num_regions; i++) { 3021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *dp = old_pages; 3031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (where[i].count) 3041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds do_region(rw, i, where + i, dp, io); 3051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 308f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen * Drop the extra reference that we were holding to avoid 3091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * the io being completed too early. 3101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 3111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dec_count(io, 0, 0); 3121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic int sync_io(unsigned int num_regions, struct io_region *where, 3151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int rw, struct dpages *dp, unsigned long *error_bits) 3161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct io io; 3181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (num_regions > 1 && rw != WRITE) { 3201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds WARN_ON(1); 3211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EIO; 3221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io.error = 0; 3251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_set(&io.count, 1); /* see dispatch_io() */ 3261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io.sleeper = current; 3271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dispatch_io(rw, num_regions, where, dp, &io, 1); 3291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (1) { 3311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds set_current_state(TASK_UNINTERRUPTIBLE); 3321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!atomic_read(&io.count) || signal_pending(current)) 3341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds break; 3351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io_schedule(); 3371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds set_current_state(TASK_RUNNING); 3391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (atomic_read(&io.count)) 3411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EINTR; 3421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *error_bits = io.error; 3441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return io.error ? -EIO : 0; 3451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic int async_io(unsigned int num_regions, struct io_region *where, int rw, 3481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct dpages *dp, io_notify_fn fn, void *context) 3491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct io *io; 3511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (num_regions > 1 && rw != WRITE) { 3531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds WARN_ON(1); 3541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fn(1, context); 3551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EIO; 3561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io = mempool_alloc(_io_pool, GFP_NOIO); 3591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io->error = 0; 3601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_set(&io->count, 1); /* see dispatch_io() */ 3611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io->sleeper = NULL; 3621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io->callback = fn; 3631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io->context = context; 3641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dispatch_io(rw, num_regions, where, dp, io, 0); 3661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 3671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint dm_io_sync(unsigned int num_regions, struct io_region *where, int rw, 3701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page_list *pl, unsigned int offset, 3711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long *error_bits) 3721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct dpages dp; 3741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds list_dp_init(&dp, pl, offset); 3751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return sync_io(num_regions, where, rw, &dp, error_bits); 3761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint dm_io_sync_bvec(unsigned int num_regions, struct io_region *where, int rw, 3791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct bio_vec *bvec, unsigned long *error_bits) 3801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct dpages dp; 3821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bvec_dp_init(&dp, bvec); 3831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return sync_io(num_regions, where, rw, &dp, error_bits); 3841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint dm_io_sync_vm(unsigned int num_regions, struct io_region *where, int rw, 3871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *data, unsigned long *error_bits) 3881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct dpages dp; 3901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds vm_dp_init(&dp, data); 3911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return sync_io(num_regions, where, rw, &dp, error_bits); 3921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint dm_io_async(unsigned int num_regions, struct io_region *where, int rw, 3951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page_list *pl, unsigned int offset, 3961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io_notify_fn fn, void *context) 3971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct dpages dp; 3991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds list_dp_init(&dp, pl, offset); 4001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return async_io(num_regions, where, rw, &dp, fn, context); 4011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint dm_io_async_bvec(unsigned int num_regions, struct io_region *where, int rw, 4041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct bio_vec *bvec, io_notify_fn fn, void *context) 4051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 4061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct dpages dp; 4071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bvec_dp_init(&dp, bvec); 4081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return async_io(num_regions, where, rw, &dp, fn, context); 4091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint dm_io_async_vm(unsigned int num_regions, struct io_region *where, int rw, 4121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *data, io_notify_fn fn, void *context) 4131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 4141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct dpages dp; 4151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds vm_dp_init(&dp, data); 4161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return async_io(num_regions, where, rw, &dp, fn, context); 4171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(dm_io_get); 4201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(dm_io_put); 4211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(dm_io_sync); 4221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(dm_io_async); 4231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(dm_io_sync_bvec); 4241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(dm_io_async_bvec); 4251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(dm_io_sync_vm); 4261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(dm_io_async_vm); 427