dm-io.c revision bf17ce3a604d943f29bf1bc1a66a4e0d2ad4ec96
11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (C) 2003 Sistina Software 3891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen * Copyright (C) 2006 Red Hat GmbH 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This file is released under the GPL. 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include "dm-io.h" 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/bio.h> 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mempool.h> 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/module.h> 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/sched.h> 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/slab.h> 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 16891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagenstruct dm_io_client { 17891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen mempool_t *pool; 18891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen struct bio_set *bios; 19891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen}; 20891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* FIXME: can we shrink this ? */ 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstruct io { 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long error; 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_t count; 251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct task_struct *sleeper; 26891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen struct dm_io_client *client; 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io_notify_fn callback; 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *context; 291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}; 301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * io contexts are only dynamically allocated for asynchronous 331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * io. Since async io is likely to be the majority of io we'll 34891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen * have the same number of io contexts as bios! (FIXME: must reduce this). 351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 36891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen 371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic unsigned int pages_to_ios(unsigned int pages) 381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 4 * pages; /* too many ? */ 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 42c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen/* 43c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen * Create a client with mempool and bioset. 44c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen */ 45c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagenstruct dm_io_client *dm_io_client_create(unsigned num_pages) 46c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 47c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen unsigned ios = pages_to_ios(num_pages); 48c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen struct dm_io_client *client; 49c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 50c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen client = kmalloc(sizeof(*client), GFP_KERNEL); 51c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen if (!client) 52c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return ERR_PTR(-ENOMEM); 53c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 54c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen client->pool = mempool_create_kmalloc_pool(ios, sizeof(struct io)); 55c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen if (!client->pool) 56c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen goto bad; 57c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 58c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen client->bios = bioset_create(16, 16); 59c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen if (!client->bios) 60c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen goto bad; 61c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 62c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return client; 63c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 64c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen bad: 65c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen if (client->pool) 66c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen mempool_destroy(client->pool); 67c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen kfree(client); 68c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return ERR_PTR(-ENOMEM); 69c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 70c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz MauelshagenEXPORT_SYMBOL(dm_io_client_create); 71c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 72c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagenint dm_io_client_resize(unsigned num_pages, struct dm_io_client *client) 73c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 74c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return mempool_resize(client->pool, pages_to_ios(num_pages), 75c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen GFP_KERNEL); 76c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 77c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz MauelshagenEXPORT_SYMBOL(dm_io_client_resize); 78c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 79c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagenvoid dm_io_client_destroy(struct dm_io_client *client) 80c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 81c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen mempool_destroy(client->pool); 82c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen bioset_free(client->bios); 83c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen kfree(client); 84c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 85c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz MauelshagenEXPORT_SYMBOL(dm_io_client_destroy); 86c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*----------------------------------------------------------------- 881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * We need to keep track of which region a bio is doing io for. 891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * In order to save a memory allocation we store this the last 901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * bvec which we know is unused (blech). 911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * XXX This is ugly and can OOPS with some configs... find another way. 921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *---------------------------------------------------------------*/ 931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic inline void bio_set_region(struct bio *bio, unsigned region) 941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 95f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen bio->bi_io_vec[bio->bi_max_vecs].bv_len = region; 961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic inline unsigned bio_get_region(struct bio *bio) 991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 100f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen return bio->bi_io_vec[bio->bi_max_vecs].bv_len; 1011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*----------------------------------------------------------------- 1041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * We need an io object to keep track of the number of bios that 1051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * have been dispatched for a particular io. 1061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *---------------------------------------------------------------*/ 1071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void dec_count(struct io *io, unsigned int region, int error) 1081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (error) 1101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds set_bit(region, &io->error); 1111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (atomic_dec_and_test(&io->count)) { 1131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (io->sleeper) 1141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds wake_up_process(io->sleeper); 1151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds else { 1171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int r = io->error; 1181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io_notify_fn fn = io->callback; 1191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *context = io->context; 1201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 121bf17ce3a604d943f29bf1bc1a66a4e0d2ad4ec96Milan Broz mempool_free(io, io->client->pool); 1221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fn(r, context); 1231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic int endio(struct bio *bio, unsigned int done, int error) 1281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 129c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen struct io *io; 130c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen unsigned region; 1311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* keep going until we've finished */ 1331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (bio->bi_size) 1341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 1; 1351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (error && bio_data_dir(bio) == READ) 1371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds zero_fill_bio(bio); 1381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 139c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen /* 140c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen * The bio destructor in bio_put() may use the io object. 141c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen */ 142c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen io = bio->bi_private; 143c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen region = bio_get_region(bio); 144c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen 145f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen bio->bi_max_vecs++; 1461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio_put(bio); 1471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 148c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen dec_count(io, region, error); 149c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen 1501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 1511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*----------------------------------------------------------------- 1541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * These little objects provide an abstraction for getting a new 1551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * destination page for io. 1561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *---------------------------------------------------------------*/ 1571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstruct dpages { 1581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void (*get_page)(struct dpages *dp, 1591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page **p, unsigned long *len, unsigned *offset); 1601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void (*next_page)(struct dpages *dp); 1611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned context_u; 1631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *context_ptr; 1641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}; 1651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 1671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Functions for getting the pages from a list. 1681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 1691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void list_get_page(struct dpages *dp, 1701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page **p, unsigned long *len, unsigned *offset) 1711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned o = dp->context_u; 1731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page_list *pl = (struct page_list *) dp->context_ptr; 1741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *p = pl->page; 1761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *len = PAGE_SIZE - o; 1771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *offset = o; 1781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void list_next_page(struct dpages *dp) 1811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page_list *pl = (struct page_list *) dp->context_ptr; 1831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr = pl->next; 1841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_u = 0; 1851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void list_dp_init(struct dpages *dp, struct page_list *pl, unsigned offset) 1881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->get_page = list_get_page; 1901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->next_page = list_next_page; 1911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_u = offset; 1921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr = pl; 1931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 1961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Functions for getting the pages from a bvec. 1971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 1981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void bvec_get_page(struct dpages *dp, 1991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page **p, unsigned long *len, unsigned *offset) 2001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct bio_vec *bvec = (struct bio_vec *) dp->context_ptr; 2021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *p = bvec->bv_page; 2031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *len = bvec->bv_len; 2041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *offset = bvec->bv_offset; 2051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void bvec_next_page(struct dpages *dp) 2081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct bio_vec *bvec = (struct bio_vec *) dp->context_ptr; 2101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr = bvec + 1; 2111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void bvec_dp_init(struct dpages *dp, struct bio_vec *bvec) 2141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->get_page = bvec_get_page; 2161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->next_page = bvec_next_page; 2171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr = bvec; 2181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 220c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen/* 221c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen * Functions for getting the pages from a VMA. 222c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen */ 2231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void vm_get_page(struct dpages *dp, 2241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page **p, unsigned long *len, unsigned *offset) 2251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *p = vmalloc_to_page(dp->context_ptr); 2271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *offset = dp->context_u; 2281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *len = PAGE_SIZE - dp->context_u; 2291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void vm_next_page(struct dpages *dp) 2321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr += PAGE_SIZE - dp->context_u; 2341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_u = 0; 2351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void vm_dp_init(struct dpages *dp, void *data) 2381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->get_page = vm_get_page; 2401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->next_page = vm_next_page; 2411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_u = ((unsigned long) data) & (PAGE_SIZE - 1); 2421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr = data; 2431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2453676347a5e216a7fec7f8eedbbcf8bed6b9c4e40Peter Osterlundstatic void dm_bio_destructor(struct bio *bio) 2463676347a5e216a7fec7f8eedbbcf8bed6b9c4e40Peter Osterlund{ 247891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen struct io *io = bio->bi_private; 248891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen 249bf17ce3a604d943f29bf1bc1a66a4e0d2ad4ec96Milan Broz bio_free(bio, io->client->bios); 2503676347a5e216a7fec7f8eedbbcf8bed6b9c4e40Peter Osterlund} 2513676347a5e216a7fec7f8eedbbcf8bed6b9c4e40Peter Osterlund 252c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen/* 253c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen * Functions for getting the pages from kernel memory. 254c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen */ 255c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagenstatic void km_get_page(struct dpages *dp, struct page **p, unsigned long *len, 256c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen unsigned *offset) 257c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 258c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen *p = virt_to_page(dp->context_ptr); 259c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen *offset = dp->context_u; 260c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen *len = PAGE_SIZE - dp->context_u; 261c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 262c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 263c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagenstatic void km_next_page(struct dpages *dp) 264c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 265c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen dp->context_ptr += PAGE_SIZE - dp->context_u; 266c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen dp->context_u = 0; 267c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 268c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 269c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagenstatic void km_dp_init(struct dpages *dp, void *data) 270c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 271c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen dp->get_page = km_get_page; 272c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen dp->next_page = km_next_page; 273c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen dp->context_u = ((unsigned long) data) & (PAGE_SIZE - 1); 274c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen dp->context_ptr = data; 275c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 276c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 2771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*----------------------------------------------------------------- 2781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * IO routines that accept a list of pages. 2791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *---------------------------------------------------------------*/ 2801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void do_region(int rw, unsigned int region, struct io_region *where, 2811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct dpages *dp, struct io *io) 2821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct bio *bio; 2841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page *page; 2851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long len; 2861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned offset; 2871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned num_bvecs; 2881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds sector_t remaining = where->count; 2891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (remaining) { 2911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 292f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen * Allocate a suitably sized-bio: we add an extra 293f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen * bvec for bio_get/set_region() and decrement bi_max_vecs 294f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen * to hide it from bio_add_page(). 2951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 296f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen num_bvecs = (remaining / (PAGE_SIZE >> SECTOR_SHIFT)) + 2; 297bf17ce3a604d943f29bf1bc1a66a4e0d2ad4ec96Milan Broz bio = bio_alloc_bioset(GFP_NOIO, num_bvecs, io->client->bios); 2981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio->bi_sector = where->sector + (where->count - remaining); 2991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio->bi_bdev = where->bdev; 3001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio->bi_end_io = endio; 3011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio->bi_private = io; 3023676347a5e216a7fec7f8eedbbcf8bed6b9c4e40Peter Osterlund bio->bi_destructor = dm_bio_destructor; 303f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen bio->bi_max_vecs--; 3041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio_set_region(bio, region); 3051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 3071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Try and add as many pages as possible. 3081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 3091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (remaining) { 3101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->get_page(dp, &page, &len, &offset); 3111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds len = min(len, to_bytes(remaining)); 3121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!bio_add_page(bio, page, len, offset)) 3131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds break; 3141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds offset = 0; 3161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds remaining -= to_sector(len); 3171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->next_page(dp); 3181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_inc(&io->count); 3211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds submit_bio(rw, bio); 3221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void dispatch_io(int rw, unsigned int num_regions, 3261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct io_region *where, struct dpages *dp, 3271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct io *io, int sync) 3281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int i; 3301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct dpages old_pages = *dp; 3311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (sync) 3331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds rw |= (1 << BIO_RW_SYNC); 3341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 3361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * For multiple regions we need to be careful to rewind 3371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * the dp object for each call to do_region. 3381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 3391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds for (i = 0; i < num_regions; i++) { 3401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *dp = old_pages; 3411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (where[i].count) 3421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds do_region(rw, i, where + i, dp, io); 3431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 346f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen * Drop the extra reference that we were holding to avoid 3471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * the io being completed too early. 3481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 3491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dec_count(io, 0, 0); 3501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 352891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagenstatic int sync_io(struct dm_io_client *client, unsigned int num_regions, 353891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen struct io_region *where, int rw, struct dpages *dp, 354891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen unsigned long *error_bits) 3551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct io io; 3571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (num_regions > 1 && rw != WRITE) { 3591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds WARN_ON(1); 3601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EIO; 3611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io.error = 0; 3641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_set(&io.count, 1); /* see dispatch_io() */ 3651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io.sleeper = current; 366891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen io.client = client; 3671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dispatch_io(rw, num_regions, where, dp, &io, 1); 3691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (1) { 3711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds set_current_state(TASK_UNINTERRUPTIBLE); 3721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!atomic_read(&io.count) || signal_pending(current)) 3741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds break; 3751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io_schedule(); 3771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds set_current_state(TASK_RUNNING); 3791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (atomic_read(&io.count)) 3811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EINTR; 3821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 383891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen if (error_bits) 384891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen *error_bits = io.error; 385891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen 3861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return io.error ? -EIO : 0; 3871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 389891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagenstatic int async_io(struct dm_io_client *client, unsigned int num_regions, 390891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen struct io_region *where, int rw, struct dpages *dp, 391891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen io_notify_fn fn, void *context) 3921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct io *io; 3941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (num_regions > 1 && rw != WRITE) { 3961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds WARN_ON(1); 3971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fn(1, context); 3981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EIO; 3991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 4001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 401bf17ce3a604d943f29bf1bc1a66a4e0d2ad4ec96Milan Broz io = mempool_alloc(client->pool, GFP_NOIO); 4021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io->error = 0; 4031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_set(&io->count, 1); /* see dispatch_io() */ 4041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io->sleeper = NULL; 405891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen io->client = client; 4061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io->callback = fn; 4071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io->context = context; 4081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dispatch_io(rw, num_regions, where, dp, io, 0); 4101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 4111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 413c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagenstatic int dp_init(struct dm_io_request *io_req, struct dpages *dp) 414c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 415c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen /* Set up dpages based on memory type */ 416c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen switch (io_req->mem.type) { 417c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen case DM_IO_PAGE_LIST: 418c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen list_dp_init(dp, io_req->mem.ptr.pl, io_req->mem.offset); 419c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen break; 420c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 421c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen case DM_IO_BVEC: 422c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen bvec_dp_init(dp, io_req->mem.ptr.bvec); 423c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen break; 424c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 425c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen case DM_IO_VMA: 426c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen vm_dp_init(dp, io_req->mem.ptr.vma); 427c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen break; 428c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 429c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen case DM_IO_KMEM: 430c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen km_dp_init(dp, io_req->mem.ptr.addr); 431c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen break; 432c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 433c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen default: 434c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return -EINVAL; 435c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen } 436c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 437c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return 0; 438c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 439c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 440c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen/* 441c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen * New collapsed (a)synchronous interface 442c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen */ 443c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagenint dm_io(struct dm_io_request *io_req, unsigned num_regions, 444c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen struct io_region *where, unsigned long *sync_error_bits) 445c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 446c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen int r; 447c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen struct dpages dp; 448c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 449c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen r = dp_init(io_req, &dp); 450c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen if (r) 451c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return r; 452c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 453c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen if (!io_req->notify.fn) 454c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return sync_io(io_req->client, num_regions, where, 455c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen io_req->bi_rw, &dp, sync_error_bits); 456c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 457c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return async_io(io_req->client, num_regions, where, io_req->bi_rw, 458c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen &dp, io_req->notify.fn, io_req->notify.context); 459c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 460c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz MauelshagenEXPORT_SYMBOL(dm_io); 461