dm-io.c revision c8b03afe3d38a635861e4bfa5c563d844e754a91
11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (C) 2003 Sistina Software 3891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen * Copyright (C) 2006 Red Hat GmbH 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This file is released under the GPL. 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include "dm-io.h" 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/bio.h> 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mempool.h> 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/module.h> 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/sched.h> 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/slab.h> 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic struct bio_set *_bios; 171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 18891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagenstruct dm_io_client { 19891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen mempool_t *pool; 20891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen struct bio_set *bios; 21891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen}; 22891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* FIXME: can we shrink this ? */ 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstruct io { 251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long error; 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_t count; 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct task_struct *sleeper; 28891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen struct dm_io_client *client; 291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io_notify_fn callback; 301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *context; 311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}; 321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * io contexts are only dynamically allocated for asynchronous 351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * io. Since async io is likely to be the majority of io we'll 36891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen * have the same number of io contexts as bios! (FIXME: must reduce this). 371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic unsigned _num_ios; 391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic mempool_t *_io_pool; 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 41891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen/* 42891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen * Temporary functions to allow old and new interfaces to co-exist. 43891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen */ 44891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagenstatic struct bio_set *bios(struct dm_io_client *client) 45891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen{ 46891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen return client ? client->bios : _bios; 47891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen} 48891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen 49891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagenstatic mempool_t *io_pool(struct dm_io_client *client) 50891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen{ 51891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen return client ? client->pool : _io_pool; 52891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen} 53891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen 541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic unsigned int pages_to_ios(unsigned int pages) 551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 4 * pages; /* too many ? */ 571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic int resize_pool(unsigned int new_ios) 601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int r = 0; 621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (_io_pool) { 641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (new_ios == 0) { 651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* free off the pool */ 661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mempool_destroy(_io_pool); 671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds _io_pool = NULL; 681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bioset_free(_bios); 691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } else { 711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* resize the pool */ 721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds r = mempool_resize(_io_pool, new_ios, GFP_KERNEL); 731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } else { 761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* create new pool */ 770eaae62abaa1ad1f231932b6cdd9fb1b91df6651Matthew Dobson _io_pool = mempool_create_kmalloc_pool(new_ios, 780eaae62abaa1ad1f231932b6cdd9fb1b91df6651Matthew Dobson sizeof(struct io)); 791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!_io_pool) 801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -ENOMEM; 811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 825972511b77809cb7c9ccdb79b825c54921c5c546Jens Axboe _bios = bioset_create(16, 16); 831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!_bios) { 841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mempool_destroy(_io_pool); 851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds _io_pool = NULL; 861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -ENOMEM; 871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!r) 911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds _num_ios = new_ios; 921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return r; 941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint dm_io_get(unsigned int num_pages) 971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return resize_pool(_num_ios + pages_to_ios(num_pages)); 991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid dm_io_put(unsigned int num_pages) 1021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds resize_pool(_num_ios - pages_to_ios(num_pages)); 1041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 106c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen/* 107c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen * Create a client with mempool and bioset. 108c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen */ 109c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagenstruct dm_io_client *dm_io_client_create(unsigned num_pages) 110c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 111c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen unsigned ios = pages_to_ios(num_pages); 112c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen struct dm_io_client *client; 113c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 114c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen client = kmalloc(sizeof(*client), GFP_KERNEL); 115c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen if (!client) 116c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return ERR_PTR(-ENOMEM); 117c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 118c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen client->pool = mempool_create_kmalloc_pool(ios, sizeof(struct io)); 119c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen if (!client->pool) 120c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen goto bad; 121c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 122c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen client->bios = bioset_create(16, 16); 123c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen if (!client->bios) 124c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen goto bad; 125c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 126c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return client; 127c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 128c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen bad: 129c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen if (client->pool) 130c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen mempool_destroy(client->pool); 131c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen kfree(client); 132c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return ERR_PTR(-ENOMEM); 133c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 134c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz MauelshagenEXPORT_SYMBOL(dm_io_client_create); 135c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 136c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagenint dm_io_client_resize(unsigned num_pages, struct dm_io_client *client) 137c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 138c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return mempool_resize(client->pool, pages_to_ios(num_pages), 139c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen GFP_KERNEL); 140c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 141c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz MauelshagenEXPORT_SYMBOL(dm_io_client_resize); 142c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 143c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagenvoid dm_io_client_destroy(struct dm_io_client *client) 144c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 145c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen mempool_destroy(client->pool); 146c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen bioset_free(client->bios); 147c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen kfree(client); 148c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 149c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz MauelshagenEXPORT_SYMBOL(dm_io_client_destroy); 150c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 1511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*----------------------------------------------------------------- 1521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * We need to keep track of which region a bio is doing io for. 1531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * In order to save a memory allocation we store this the last 1541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * bvec which we know is unused (blech). 1551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * XXX This is ugly and can OOPS with some configs... find another way. 1561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *---------------------------------------------------------------*/ 1571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic inline void bio_set_region(struct bio *bio, unsigned region) 1581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 159f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen bio->bi_io_vec[bio->bi_max_vecs].bv_len = region; 1601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic inline unsigned bio_get_region(struct bio *bio) 1631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 164f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen return bio->bi_io_vec[bio->bi_max_vecs].bv_len; 1651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*----------------------------------------------------------------- 1681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * We need an io object to keep track of the number of bios that 1691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * have been dispatched for a particular io. 1701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *---------------------------------------------------------------*/ 1711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void dec_count(struct io *io, unsigned int region, int error) 1721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (error) 1741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds set_bit(region, &io->error); 1751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (atomic_dec_and_test(&io->count)) { 1771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (io->sleeper) 1781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds wake_up_process(io->sleeper); 1791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds else { 1811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int r = io->error; 1821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io_notify_fn fn = io->callback; 1831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *context = io->context; 1841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 185891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen mempool_free(io, io_pool(io->client)); 1861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fn(r, context); 1871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic int endio(struct bio *bio, unsigned int done, int error) 1921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 193c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen struct io *io; 194c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen unsigned region; 1951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* keep going until we've finished */ 1971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (bio->bi_size) 1981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 1; 1991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (error && bio_data_dir(bio) == READ) 2011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds zero_fill_bio(bio); 2021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 203c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen /* 204c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen * The bio destructor in bio_put() may use the io object. 205c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen */ 206c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen io = bio->bi_private; 207c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen region = bio_get_region(bio); 208c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen 209f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen bio->bi_max_vecs++; 2101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio_put(bio); 2111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 212c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen dec_count(io, region, error); 213c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen 2141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 2151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*----------------------------------------------------------------- 2181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * These little objects provide an abstraction for getting a new 2191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * destination page for io. 2201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *---------------------------------------------------------------*/ 2211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstruct dpages { 2221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void (*get_page)(struct dpages *dp, 2231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page **p, unsigned long *len, unsigned *offset); 2241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void (*next_page)(struct dpages *dp); 2251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned context_u; 2271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *context_ptr; 2281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}; 2291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 2311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Functions for getting the pages from a list. 2321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 2331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void list_get_page(struct dpages *dp, 2341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page **p, unsigned long *len, unsigned *offset) 2351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned o = dp->context_u; 2371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page_list *pl = (struct page_list *) dp->context_ptr; 2381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *p = pl->page; 2401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *len = PAGE_SIZE - o; 2411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *offset = o; 2421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void list_next_page(struct dpages *dp) 2451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page_list *pl = (struct page_list *) dp->context_ptr; 2471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr = pl->next; 2481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_u = 0; 2491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void list_dp_init(struct dpages *dp, struct page_list *pl, unsigned offset) 2521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->get_page = list_get_page; 2541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->next_page = list_next_page; 2551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_u = offset; 2561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr = pl; 2571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 2601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Functions for getting the pages from a bvec. 2611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 2621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void bvec_get_page(struct dpages *dp, 2631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page **p, unsigned long *len, unsigned *offset) 2641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct bio_vec *bvec = (struct bio_vec *) dp->context_ptr; 2661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *p = bvec->bv_page; 2671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *len = bvec->bv_len; 2681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *offset = bvec->bv_offset; 2691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void bvec_next_page(struct dpages *dp) 2721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct bio_vec *bvec = (struct bio_vec *) dp->context_ptr; 2741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr = bvec + 1; 2751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void bvec_dp_init(struct dpages *dp, struct bio_vec *bvec) 2781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->get_page = bvec_get_page; 2801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->next_page = bvec_next_page; 2811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr = bvec; 2821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 284c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen/* 285c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen * Functions for getting the pages from a VMA. 286c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen */ 2871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void vm_get_page(struct dpages *dp, 2881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page **p, unsigned long *len, unsigned *offset) 2891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *p = vmalloc_to_page(dp->context_ptr); 2911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *offset = dp->context_u; 2921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *len = PAGE_SIZE - dp->context_u; 2931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void vm_next_page(struct dpages *dp) 2961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr += PAGE_SIZE - dp->context_u; 2981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_u = 0; 2991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void vm_dp_init(struct dpages *dp, void *data) 3021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->get_page = vm_get_page; 3041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->next_page = vm_next_page; 3051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_u = ((unsigned long) data) & (PAGE_SIZE - 1); 3061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr = data; 3071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3093676347a5e216a7fec7f8eedbbcf8bed6b9c4e40Peter Osterlundstatic void dm_bio_destructor(struct bio *bio) 3103676347a5e216a7fec7f8eedbbcf8bed6b9c4e40Peter Osterlund{ 311891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen struct io *io = bio->bi_private; 312891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen 313891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen bio_free(bio, bios(io->client)); 3143676347a5e216a7fec7f8eedbbcf8bed6b9c4e40Peter Osterlund} 3153676347a5e216a7fec7f8eedbbcf8bed6b9c4e40Peter Osterlund 316c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen/* 317c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen * Functions for getting the pages from kernel memory. 318c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen */ 319c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagenstatic void km_get_page(struct dpages *dp, struct page **p, unsigned long *len, 320c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen unsigned *offset) 321c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 322c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen *p = virt_to_page(dp->context_ptr); 323c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen *offset = dp->context_u; 324c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen *len = PAGE_SIZE - dp->context_u; 325c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 326c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 327c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagenstatic void km_next_page(struct dpages *dp) 328c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 329c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen dp->context_ptr += PAGE_SIZE - dp->context_u; 330c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen dp->context_u = 0; 331c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 332c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 333c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagenstatic void km_dp_init(struct dpages *dp, void *data) 334c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 335c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen dp->get_page = km_get_page; 336c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen dp->next_page = km_next_page; 337c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen dp->context_u = ((unsigned long) data) & (PAGE_SIZE - 1); 338c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen dp->context_ptr = data; 339c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 340c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 3411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*----------------------------------------------------------------- 3421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * IO routines that accept a list of pages. 3431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *---------------------------------------------------------------*/ 3441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void do_region(int rw, unsigned int region, struct io_region *where, 3451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct dpages *dp, struct io *io) 3461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct bio *bio; 3481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page *page; 3491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long len; 3501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned offset; 3511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned num_bvecs; 3521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds sector_t remaining = where->count; 3531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (remaining) { 3551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 356f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen * Allocate a suitably sized-bio: we add an extra 357f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen * bvec for bio_get/set_region() and decrement bi_max_vecs 358f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen * to hide it from bio_add_page(). 3591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 360f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen num_bvecs = (remaining / (PAGE_SIZE >> SECTOR_SHIFT)) + 2; 361891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen bio = bio_alloc_bioset(GFP_NOIO, num_bvecs, bios(io->client)); 3621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio->bi_sector = where->sector + (where->count - remaining); 3631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio->bi_bdev = where->bdev; 3641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio->bi_end_io = endio; 3651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio->bi_private = io; 3663676347a5e216a7fec7f8eedbbcf8bed6b9c4e40Peter Osterlund bio->bi_destructor = dm_bio_destructor; 367f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen bio->bi_max_vecs--; 3681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio_set_region(bio, region); 3691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 3711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Try and add as many pages as possible. 3721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 3731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (remaining) { 3741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->get_page(dp, &page, &len, &offset); 3751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds len = min(len, to_bytes(remaining)); 3761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!bio_add_page(bio, page, len, offset)) 3771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds break; 3781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds offset = 0; 3801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds remaining -= to_sector(len); 3811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->next_page(dp); 3821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_inc(&io->count); 3851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds submit_bio(rw, bio); 3861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void dispatch_io(int rw, unsigned int num_regions, 3901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct io_region *where, struct dpages *dp, 3911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct io *io, int sync) 3921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int i; 3941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct dpages old_pages = *dp; 3951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (sync) 3971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds rw |= (1 << BIO_RW_SYNC); 3981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 4001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * For multiple regions we need to be careful to rewind 4011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * the dp object for each call to do_region. 4021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 4031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds for (i = 0; i < num_regions; i++) { 4041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *dp = old_pages; 4051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (where[i].count) 4061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds do_region(rw, i, where + i, dp, io); 4071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 4081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 410f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen * Drop the extra reference that we were holding to avoid 4111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * the io being completed too early. 4121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 4131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dec_count(io, 0, 0); 4141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 416891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagenstatic int sync_io(struct dm_io_client *client, unsigned int num_regions, 417891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen struct io_region *where, int rw, struct dpages *dp, 418891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen unsigned long *error_bits) 4191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 4201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct io io; 4211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (num_regions > 1 && rw != WRITE) { 4231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds WARN_ON(1); 4241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EIO; 4251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 4261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io.error = 0; 4281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_set(&io.count, 1); /* see dispatch_io() */ 4291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io.sleeper = current; 430891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen io.client = client; 4311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dispatch_io(rw, num_regions, where, dp, &io, 1); 4331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (1) { 4351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds set_current_state(TASK_UNINTERRUPTIBLE); 4361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!atomic_read(&io.count) || signal_pending(current)) 4381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds break; 4391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io_schedule(); 4411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 4421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds set_current_state(TASK_RUNNING); 4431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (atomic_read(&io.count)) 4451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EINTR; 4461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 447891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen if (error_bits) 448891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen *error_bits = io.error; 449891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen 4501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return io.error ? -EIO : 0; 4511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 453891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagenstatic int async_io(struct dm_io_client *client, unsigned int num_regions, 454891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen struct io_region *where, int rw, struct dpages *dp, 455891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen io_notify_fn fn, void *context) 4561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 4571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct io *io; 4581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (num_regions > 1 && rw != WRITE) { 4601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds WARN_ON(1); 4611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fn(1, context); 4621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EIO; 4631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 4641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 465891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen io = mempool_alloc(io_pool(client), GFP_NOIO); 4661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io->error = 0; 4671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_set(&io->count, 1); /* see dispatch_io() */ 4681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io->sleeper = NULL; 469891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen io->client = client; 4701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io->callback = fn; 4711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io->context = context; 4721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dispatch_io(rw, num_regions, where, dp, io, 0); 4741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 4751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint dm_io_sync(unsigned int num_regions, struct io_region *where, int rw, 4781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page_list *pl, unsigned int offset, 4791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long *error_bits) 4801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 4811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct dpages dp; 4821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds list_dp_init(&dp, pl, offset); 483891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen return sync_io(NULL, num_regions, where, rw, &dp, error_bits); 4841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint dm_io_sync_bvec(unsigned int num_regions, struct io_region *where, int rw, 4871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct bio_vec *bvec, unsigned long *error_bits) 4881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 4891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct dpages dp; 4901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bvec_dp_init(&dp, bvec); 491891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen return sync_io(NULL, num_regions, where, rw, &dp, error_bits); 4921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint dm_io_sync_vm(unsigned int num_regions, struct io_region *where, int rw, 4951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *data, unsigned long *error_bits) 4961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 4971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct dpages dp; 4981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds vm_dp_init(&dp, data); 499891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen return sync_io(NULL, num_regions, where, rw, &dp, error_bits); 5001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 5011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint dm_io_async(unsigned int num_regions, struct io_region *where, int rw, 5031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page_list *pl, unsigned int offset, 5041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io_notify_fn fn, void *context) 5051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 5061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct dpages dp; 5071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds list_dp_init(&dp, pl, offset); 508891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen return async_io(NULL, num_regions, where, rw, &dp, fn, context); 5091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 5101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint dm_io_async_bvec(unsigned int num_regions, struct io_region *where, int rw, 5121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct bio_vec *bvec, io_notify_fn fn, void *context) 5131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 5141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct dpages dp; 5151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bvec_dp_init(&dp, bvec); 516891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen return async_io(NULL, num_regions, where, rw, &dp, fn, context); 5171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 5181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint dm_io_async_vm(unsigned int num_regions, struct io_region *where, int rw, 5201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *data, io_notify_fn fn, void *context) 5211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 5221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct dpages dp; 5231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds vm_dp_init(&dp, data); 524891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen return async_io(NULL, num_regions, where, rw, &dp, fn, context); 5251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 5261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 527c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagenstatic int dp_init(struct dm_io_request *io_req, struct dpages *dp) 528c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 529c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen /* Set up dpages based on memory type */ 530c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen switch (io_req->mem.type) { 531c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen case DM_IO_PAGE_LIST: 532c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen list_dp_init(dp, io_req->mem.ptr.pl, io_req->mem.offset); 533c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen break; 534c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 535c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen case DM_IO_BVEC: 536c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen bvec_dp_init(dp, io_req->mem.ptr.bvec); 537c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen break; 538c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 539c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen case DM_IO_VMA: 540c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen vm_dp_init(dp, io_req->mem.ptr.vma); 541c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen break; 542c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 543c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen case DM_IO_KMEM: 544c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen km_dp_init(dp, io_req->mem.ptr.addr); 545c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen break; 546c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 547c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen default: 548c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return -EINVAL; 549c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen } 550c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 551c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return 0; 552c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 553c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 554c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen/* 555c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen * New collapsed (a)synchronous interface 556c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen */ 557c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagenint dm_io(struct dm_io_request *io_req, unsigned num_regions, 558c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen struct io_region *where, unsigned long *sync_error_bits) 559c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 560c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen int r; 561c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen struct dpages dp; 562c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 563c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen r = dp_init(io_req, &dp); 564c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen if (r) 565c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return r; 566c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 567c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen if (!io_req->notify.fn) 568c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return sync_io(io_req->client, num_regions, where, 569c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen io_req->bi_rw, &dp, sync_error_bits); 570c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 571c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return async_io(io_req->client, num_regions, where, io_req->bi_rw, 572c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen &dp, io_req->notify.fn, io_req->notify.context); 573c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 574c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz MauelshagenEXPORT_SYMBOL(dm_io); 575c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 5761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(dm_io_get); 5771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(dm_io_put); 5781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(dm_io_sync); 5791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(dm_io_async); 5801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(dm_io_sync_bvec); 5811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(dm_io_async_bvec); 5821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(dm_io_sync_vm); 5831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(dm_io_async_vm); 584