11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (C) 2003 Sistina Software 3891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen * Copyright (C) 2006 Red Hat GmbH 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This file is released under the GPL. 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8952b355760c196ec014dd0b6878f85a11496e3daMikulas Patocka#include "dm.h" 9952b355760c196ec014dd0b6878f85a11496e3daMikulas Patocka 10586e80e6ee0d137c7d79fbae183bb37bc60ee97eMikulas Patocka#include <linux/device-mapper.h> 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/bio.h> 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mempool.h> 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/module.h> 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/sched.h> 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/slab.h> 17a765e20eeb423d0fa6a02ffab51141e53bbd93cbAlasdair G Kergon#include <linux/dm-io.h> 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 19f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka#define DM_MSG_PREFIX "io" 20f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka 21f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka#define DM_IO_MAX_REGIONS BITS_PER_LONG 22bda8efec5c706a672e0714d341a342e811f0262aMikulas Patocka#define MIN_IOS 16 23bda8efec5c706a672e0714d341a342e811f0262aMikulas Patocka#define MIN_BIOS 16 24f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka 25891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagenstruct dm_io_client { 26891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen mempool_t *pool; 27891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen struct bio_set *bios; 28891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen}; 29891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen 30f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka/* 31f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka * Aligning 'struct io' reduces the number of bits required to store 32f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka * its address. Refer to store_io_and_region_in_bio() below. 33f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka */ 341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstruct io { 35e01fd7eeb00f8078103f4ed3e8ef64474c11f300Alasdair G Kergon unsigned long error_bits; 361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_t count; 371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct task_struct *sleeper; 38891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen struct dm_io_client *client; 391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io_notify_fn callback; 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *context; 41bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka void *vma_invalidate_address; 42bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka unsigned long vma_invalidate_size; 43f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka} __attribute__((aligned(DM_IO_MAX_REGIONS))); 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 45952b355760c196ec014dd0b6878f85a11496e3daMikulas Patockastatic struct kmem_cache *_dm_io_cache; 46952b355760c196ec014dd0b6878f85a11496e3daMikulas Patocka 471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 48c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen * Create a client with mempool and bioset. 49c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen */ 50bda8efec5c706a672e0714d341a342e811f0262aMikulas Patockastruct dm_io_client *dm_io_client_create(void) 51c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 52c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen struct dm_io_client *client; 53c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 54c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen client = kmalloc(sizeof(*client), GFP_KERNEL); 55c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen if (!client) 56c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return ERR_PTR(-ENOMEM); 57c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 58bda8efec5c706a672e0714d341a342e811f0262aMikulas Patocka client->pool = mempool_create_slab_pool(MIN_IOS, _dm_io_cache); 59c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen if (!client->pool) 60c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen goto bad; 61c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 62bda8efec5c706a672e0714d341a342e811f0262aMikulas Patocka client->bios = bioset_create(MIN_BIOS, 0); 63c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen if (!client->bios) 64c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen goto bad; 65c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 66c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return client; 67c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 68c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen bad: 69c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen if (client->pool) 70c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen mempool_destroy(client->pool); 71c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen kfree(client); 72c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return ERR_PTR(-ENOMEM); 73c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 74c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz MauelshagenEXPORT_SYMBOL(dm_io_client_create); 75c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 76c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagenvoid dm_io_client_destroy(struct dm_io_client *client) 77c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 78c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen mempool_destroy(client->pool); 79c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen bioset_free(client->bios); 80c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen kfree(client); 81c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 82c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz MauelshagenEXPORT_SYMBOL(dm_io_client_destroy); 83c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*----------------------------------------------------------------- 851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * We need to keep track of which region a bio is doing io for. 86f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka * To avoid a memory allocation to store just 5 or 6 bits, we 87f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka * ensure the 'struct io' pointer is aligned so enough low bits are 88f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka * always zero and then combine it with the region number directly in 89f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka * bi_private. 901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *---------------------------------------------------------------*/ 91f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patockastatic void store_io_and_region_in_bio(struct bio *bio, struct io *io, 92f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka unsigned region) 931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 94f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka if (unlikely(!IS_ALIGNED((unsigned long)io, DM_IO_MAX_REGIONS))) { 95f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka DMCRIT("Unaligned struct io pointer %p", io); 96f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka BUG(); 97f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka } 98f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka 99f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka bio->bi_private = (void *)((unsigned long)io | region); 1001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 102f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patockastatic void retrieve_io_and_region_from_bio(struct bio *bio, struct io **io, 103f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka unsigned *region) 1041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 105f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka unsigned long val = (unsigned long)bio->bi_private; 106f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka 107f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka *io = (void *)(val & -(unsigned long)DM_IO_MAX_REGIONS); 108f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka *region = val & (DM_IO_MAX_REGIONS - 1); 1091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*----------------------------------------------------------------- 1121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * We need an io object to keep track of the number of bios that 1131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * have been dispatched for a particular io. 1141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *---------------------------------------------------------------*/ 1151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void dec_count(struct io *io, unsigned int region, int error) 1161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 117d87f4c14f27dc82d215108d8392a7d26687148a1Tejun Heo if (error) 118e01fd7eeb00f8078103f4ed3e8ef64474c11f300Alasdair G Kergon set_bit(region, &io->error_bits); 1191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (atomic_dec_and_test(&io->count)) { 121bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka if (io->vma_invalidate_size) 122bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka invalidate_kernel_vmap_range(io->vma_invalidate_address, 123bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka io->vma_invalidate_size); 124bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka 1251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (io->sleeper) 1261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds wake_up_process(io->sleeper); 1271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds else { 129e01fd7eeb00f8078103f4ed3e8ef64474c11f300Alasdair G Kergon unsigned long r = io->error_bits; 1301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io_notify_fn fn = io->callback; 1311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *context = io->context; 1321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 133bf17ce3a604d943f29bf1bc1a66a4e0d2ad4ec96Milan Broz mempool_free(io, io->client->pool); 1341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fn(r, context); 1351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1396712ecf8f648118c3363c142196418f89a510b90NeilBrownstatic void endio(struct bio *bio, int error) 1401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 141c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen struct io *io; 142c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen unsigned region; 1431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (error && bio_data_dir(bio) == READ) 1451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds zero_fill_bio(bio); 1461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 147c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen /* 148c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen * The bio destructor in bio_put() may use the io object. 149c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen */ 150f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka retrieve_io_and_region_from_bio(bio, &io, ®ion); 151c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen 1521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio_put(bio); 1531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 154c897feb3dcf3c3300849056ee82b01df7bf66d3cHeinz Mauelshagen dec_count(io, region, error); 1551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*----------------------------------------------------------------- 1581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * These little objects provide an abstraction for getting a new 1591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * destination page for io. 1601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *---------------------------------------------------------------*/ 1611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstruct dpages { 1621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void (*get_page)(struct dpages *dp, 1631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page **p, unsigned long *len, unsigned *offset); 1641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void (*next_page)(struct dpages *dp); 1651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned context_u; 1671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *context_ptr; 168bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka 169bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka void *vma_invalidate_address; 170bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka unsigned long vma_invalidate_size; 1711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}; 1721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 1741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Functions for getting the pages from a list. 1751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 1761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void list_get_page(struct dpages *dp, 1771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page **p, unsigned long *len, unsigned *offset) 1781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned o = dp->context_u; 1801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page_list *pl = (struct page_list *) dp->context_ptr; 1811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *p = pl->page; 1831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *len = PAGE_SIZE - o; 1841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *offset = o; 1851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void list_next_page(struct dpages *dp) 1881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page_list *pl = (struct page_list *) dp->context_ptr; 1901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr = pl->next; 1911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_u = 0; 1921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void list_dp_init(struct dpages *dp, struct page_list *pl, unsigned offset) 1951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->get_page = list_get_page; 1971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->next_page = list_next_page; 1981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_u = offset; 1991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr = pl; 2001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 2031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Functions for getting the pages from a bvec. 2041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 2051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void bvec_get_page(struct dpages *dp, 2061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page **p, unsigned long *len, unsigned *offset) 2071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct bio_vec *bvec = (struct bio_vec *) dp->context_ptr; 2091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *p = bvec->bv_page; 2101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *len = bvec->bv_len; 2111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *offset = bvec->bv_offset; 2121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void bvec_next_page(struct dpages *dp) 2151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct bio_vec *bvec = (struct bio_vec *) dp->context_ptr; 2171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr = bvec + 1; 2181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void bvec_dp_init(struct dpages *dp, struct bio_vec *bvec) 2211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->get_page = bvec_get_page; 2231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->next_page = bvec_next_page; 2241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr = bvec; 2251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 227c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen/* 228c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen * Functions for getting the pages from a VMA. 229c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen */ 2301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void vm_get_page(struct dpages *dp, 2311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page **p, unsigned long *len, unsigned *offset) 2321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *p = vmalloc_to_page(dp->context_ptr); 2341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *offset = dp->context_u; 2351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *len = PAGE_SIZE - dp->context_u; 2361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void vm_next_page(struct dpages *dp) 2391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr += PAGE_SIZE - dp->context_u; 2411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_u = 0; 2421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void vm_dp_init(struct dpages *dp, void *data) 2451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->get_page = vm_get_page; 2471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->next_page = vm_next_page; 2481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_u = ((unsigned long) data) & (PAGE_SIZE - 1); 2491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->context_ptr = data; 2501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2523676347a5e216a7fec7f8eedbbcf8bed6b9c4e40Peter Osterlundstatic void dm_bio_destructor(struct bio *bio) 2533676347a5e216a7fec7f8eedbbcf8bed6b9c4e40Peter Osterlund{ 254f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka unsigned region; 255f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka struct io *io; 256f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka 257f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka retrieve_io_and_region_from_bio(bio, &io, ®ion); 258891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen 259bf17ce3a604d943f29bf1bc1a66a4e0d2ad4ec96Milan Broz bio_free(bio, io->client->bios); 2603676347a5e216a7fec7f8eedbbcf8bed6b9c4e40Peter Osterlund} 2613676347a5e216a7fec7f8eedbbcf8bed6b9c4e40Peter Osterlund 262c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen/* 263c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen * Functions for getting the pages from kernel memory. 264c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen */ 265c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagenstatic void km_get_page(struct dpages *dp, struct page **p, unsigned long *len, 266c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen unsigned *offset) 267c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 268c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen *p = virt_to_page(dp->context_ptr); 269c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen *offset = dp->context_u; 270c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen *len = PAGE_SIZE - dp->context_u; 271c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 272c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 273c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagenstatic void km_next_page(struct dpages *dp) 274c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 275c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen dp->context_ptr += PAGE_SIZE - dp->context_u; 276c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen dp->context_u = 0; 277c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 278c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 279c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagenstatic void km_dp_init(struct dpages *dp, void *data) 280c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 281c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen dp->get_page = km_get_page; 282c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen dp->next_page = km_next_page; 283c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen dp->context_u = ((unsigned long) data) & (PAGE_SIZE - 1); 284c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen dp->context_ptr = data; 285c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 286c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 2871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*----------------------------------------------------------------- 2881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * IO routines that accept a list of pages. 2891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *---------------------------------------------------------------*/ 29022a1ceb1e6a7fbce95a1531ff10bb4fb036d4a37Heinz Mauelshagenstatic void do_region(int rw, unsigned region, struct dm_io_region *where, 2911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct dpages *dp, struct io *io) 2921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct bio *bio; 2941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page *page; 2951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long len; 2961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned offset; 2971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned num_bvecs; 2981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds sector_t remaining = where->count; 2990c535e0d6f463365c29623350dbd91642363c39bMilan Broz struct request_queue *q = bdev_get_queue(where->bdev); 3000c535e0d6f463365c29623350dbd91642363c39bMilan Broz sector_t discard_sectors; 3011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 30212fc0f49dc994d8d90dcf3df13f5b1ee5441288dMikulas Patocka /* 303d87f4c14f27dc82d215108d8392a7d26687148a1Tejun Heo * where->count may be zero if rw holds a flush and we need to 304d87f4c14f27dc82d215108d8392a7d26687148a1Tejun Heo * send a zero-sized flush. 30512fc0f49dc994d8d90dcf3df13f5b1ee5441288dMikulas Patocka */ 30612fc0f49dc994d8d90dcf3df13f5b1ee5441288dMikulas Patocka do { 3071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 308f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka * Allocate a suitably sized-bio. 3091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 3100c535e0d6f463365c29623350dbd91642363c39bMilan Broz if (rw & REQ_DISCARD) 3110c535e0d6f463365c29623350dbd91642363c39bMilan Broz num_bvecs = 1; 3120c535e0d6f463365c29623350dbd91642363c39bMilan Broz else 3130c535e0d6f463365c29623350dbd91642363c39bMilan Broz num_bvecs = min_t(int, bio_get_nr_vecs(where->bdev), 3140c535e0d6f463365c29623350dbd91642363c39bMilan Broz dm_sector_div_up(remaining, (PAGE_SIZE >> SECTOR_SHIFT))); 3150c535e0d6f463365c29623350dbd91642363c39bMilan Broz 316bf17ce3a604d943f29bf1bc1a66a4e0d2ad4ec96Milan Broz bio = bio_alloc_bioset(GFP_NOIO, num_bvecs, io->client->bios); 3171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio->bi_sector = where->sector + (where->count - remaining); 3181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio->bi_bdev = where->bdev; 3191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds bio->bi_end_io = endio; 3203676347a5e216a7fec7f8eedbbcf8bed6b9c4e40Peter Osterlund bio->bi_destructor = dm_bio_destructor; 321f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka store_io_and_region_in_bio(bio, io, region); 3221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3230c535e0d6f463365c29623350dbd91642363c39bMilan Broz if (rw & REQ_DISCARD) { 3240c535e0d6f463365c29623350dbd91642363c39bMilan Broz discard_sectors = min_t(sector_t, q->limits.max_discard_sectors, remaining); 3250c535e0d6f463365c29623350dbd91642363c39bMilan Broz bio->bi_size = discard_sectors << SECTOR_SHIFT; 3260c535e0d6f463365c29623350dbd91642363c39bMilan Broz remaining -= discard_sectors; 3270c535e0d6f463365c29623350dbd91642363c39bMilan Broz } else while (remaining) { 3280c535e0d6f463365c29623350dbd91642363c39bMilan Broz /* 3290c535e0d6f463365c29623350dbd91642363c39bMilan Broz * Try and add as many pages as possible. 3300c535e0d6f463365c29623350dbd91642363c39bMilan Broz */ 3311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->get_page(dp, &page, &len, &offset); 3321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds len = min(len, to_bytes(remaining)); 3331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!bio_add_page(bio, page, len, offset)) 3341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds break; 3351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds offset = 0; 3371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds remaining -= to_sector(len); 3381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dp->next_page(dp); 3391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_inc(&io->count); 3421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds submit_bio(rw, bio); 34312fc0f49dc994d8d90dcf3df13f5b1ee5441288dMikulas Patocka } while (remaining); 3441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void dispatch_io(int rw, unsigned int num_regions, 34722a1ceb1e6a7fbce95a1531ff10bb4fb036d4a37Heinz Mauelshagen struct dm_io_region *where, struct dpages *dp, 3481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct io *io, int sync) 3491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int i; 3511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct dpages old_pages = *dp; 3521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 353f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka BUG_ON(num_regions > DM_IO_MAX_REGIONS); 354f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka 3551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (sync) 356721a9602e6607417c6bc15b18e97a2f35266c690Jens Axboe rw |= REQ_SYNC; 3571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 3591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * For multiple regions we need to be careful to rewind 3601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * the dp object for each call to do_region. 3611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 3621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds for (i = 0; i < num_regions; i++) { 3631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *dp = old_pages; 364d87f4c14f27dc82d215108d8392a7d26687148a1Tejun Heo if (where[i].count || (rw & REQ_FLUSH)) 3651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds do_region(rw, i, where + i, dp, io); 3661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 369f00b16ad665a9b489d46f612679181f3f914917bHeinz Mauelshagen * Drop the extra reference that we were holding to avoid 3701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * the io being completed too early. 3711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 3721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dec_count(io, 0, 0); 3731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 375891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagenstatic int sync_io(struct dm_io_client *client, unsigned int num_regions, 37622a1ceb1e6a7fbce95a1531ff10bb4fb036d4a37Heinz Mauelshagen struct dm_io_region *where, int rw, struct dpages *dp, 377891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen unsigned long *error_bits) 3781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 379f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka /* 380f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka * gcc <= 4.3 can't do the alignment for stack variables, so we must 381f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka * align it on our own. 382f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka * volatile prevents the optimizer from removing or reusing 383f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka * "io_" field from the stack frame (allowed in ANSI C). 384f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka */ 385f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka volatile char io_[sizeof(struct io) + __alignof__(struct io) - 1]; 386f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka struct io *io = (struct io *)PTR_ALIGN(&io_, __alignof__(struct io)); 3871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3887ff14a36159d947872870e7a3e9dcaebc46b23ebMikulas Patocka if (num_regions > 1 && (rw & RW_MASK) != WRITE) { 3891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds WARN_ON(1); 3901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EIO; 3911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 393f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka io->error_bits = 0; 394f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka atomic_set(&io->count, 1); /* see dispatch_io() */ 395f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka io->sleeper = current; 396f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka io->client = client; 3971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 398bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka io->vma_invalidate_address = dp->vma_invalidate_address; 399bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka io->vma_invalidate_size = dp->vma_invalidate_size; 400bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka 401f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka dispatch_io(rw, num_regions, where, dp, io, 1); 4021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (1) { 4041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds set_current_state(TASK_UNINTERRUPTIBLE); 4051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 406f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka if (!atomic_read(&io->count)) 4071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds break; 4081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io_schedule(); 4101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 4111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds set_current_state(TASK_RUNNING); 4121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 413891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen if (error_bits) 414f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka *error_bits = io->error_bits; 415891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen 416f1e539874655ae9e74c1644fd54133b19f1b14e2Mikulas Patocka return io->error_bits ? -EIO : 0; 4171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 419891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagenstatic int async_io(struct dm_io_client *client, unsigned int num_regions, 42022a1ceb1e6a7fbce95a1531ff10bb4fb036d4a37Heinz Mauelshagen struct dm_io_region *where, int rw, struct dpages *dp, 421891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen io_notify_fn fn, void *context) 4221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 4231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct io *io; 4241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4257ff14a36159d947872870e7a3e9dcaebc46b23ebMikulas Patocka if (num_regions > 1 && (rw & RW_MASK) != WRITE) { 4261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds WARN_ON(1); 4271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fn(1, context); 4281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EIO; 4291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 4301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 431bf17ce3a604d943f29bf1bc1a66a4e0d2ad4ec96Milan Broz io = mempool_alloc(client->pool, GFP_NOIO); 432e01fd7eeb00f8078103f4ed3e8ef64474c11f300Alasdair G Kergon io->error_bits = 0; 4331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_set(&io->count, 1); /* see dispatch_io() */ 4341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io->sleeper = NULL; 435891ce207011d3d9219f79fd5114c8594bbacc653Heinz Mauelshagen io->client = client; 4361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io->callback = fn; 4371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds io->context = context; 4381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 439bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka io->vma_invalidate_address = dp->vma_invalidate_address; 440bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka io->vma_invalidate_size = dp->vma_invalidate_size; 441bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka 4421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dispatch_io(rw, num_regions, where, dp, io, 0); 4431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 4441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 446bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patockastatic int dp_init(struct dm_io_request *io_req, struct dpages *dp, 447bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka unsigned long size) 448c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 449c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen /* Set up dpages based on memory type */ 450bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka 451bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka dp->vma_invalidate_address = NULL; 452bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka dp->vma_invalidate_size = 0; 453bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka 454c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen switch (io_req->mem.type) { 455c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen case DM_IO_PAGE_LIST: 456c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen list_dp_init(dp, io_req->mem.ptr.pl, io_req->mem.offset); 457c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen break; 458c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 459c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen case DM_IO_BVEC: 460c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen bvec_dp_init(dp, io_req->mem.ptr.bvec); 461c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen break; 462c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 463c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen case DM_IO_VMA: 464bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka flush_kernel_vmap_range(io_req->mem.ptr.vma, size); 465bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka if ((io_req->bi_rw & RW_MASK) == READ) { 466bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka dp->vma_invalidate_address = io_req->mem.ptr.vma; 467bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka dp->vma_invalidate_size = size; 468bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka } 469c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen vm_dp_init(dp, io_req->mem.ptr.vma); 470c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen break; 471c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 472c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen case DM_IO_KMEM: 473c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen km_dp_init(dp, io_req->mem.ptr.addr); 474c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen break; 475c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 476c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen default: 477c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return -EINVAL; 478c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen } 479c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 480c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return 0; 481c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 482c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 483c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen/* 4847ff14a36159d947872870e7a3e9dcaebc46b23ebMikulas Patocka * New collapsed (a)synchronous interface. 4857ff14a36159d947872870e7a3e9dcaebc46b23ebMikulas Patocka * 4867ff14a36159d947872870e7a3e9dcaebc46b23ebMikulas Patocka * If the IO is asynchronous (i.e. it has notify.fn), you must either unplug 4877b6d91daee5cac6402186ff224c3af39d79f4a0eChristoph Hellwig * the queue with blk_unplug() some time later or set REQ_SYNC in 4887b6d91daee5cac6402186ff224c3af39d79f4a0eChristoph Hellwigio_req->bi_rw. If you fail to do one of these, the IO will be submitted to 4897ff14a36159d947872870e7a3e9dcaebc46b23ebMikulas Patocka * the disk after q->unplug_delay, which defaults to 3ms in blk-settings.c. 490c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen */ 491c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagenint dm_io(struct dm_io_request *io_req, unsigned num_regions, 49222a1ceb1e6a7fbce95a1531ff10bb4fb036d4a37Heinz Mauelshagen struct dm_io_region *where, unsigned long *sync_error_bits) 493c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen{ 494c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen int r; 495c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen struct dpages dp; 496c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 497bb91bc7bacb906c9f3a9b22744c53fa7564b51baMikulas Patocka r = dp_init(io_req, &dp, (unsigned long)where->count << SECTOR_SHIFT); 498c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen if (r) 499c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return r; 500c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 501c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen if (!io_req->notify.fn) 502c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return sync_io(io_req->client, num_regions, where, 503c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen io_req->bi_rw, &dp, sync_error_bits); 504c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen 505c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen return async_io(io_req->client, num_regions, where, io_req->bi_rw, 506c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen &dp, io_req->notify.fn, io_req->notify.context); 507c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz Mauelshagen} 508c8b03afe3d38a635861e4bfa5c563d844e754a91Heinz MauelshagenEXPORT_SYMBOL(dm_io); 509952b355760c196ec014dd0b6878f85a11496e3daMikulas Patocka 510952b355760c196ec014dd0b6878f85a11496e3daMikulas Patockaint __init dm_io_init(void) 511952b355760c196ec014dd0b6878f85a11496e3daMikulas Patocka{ 512952b355760c196ec014dd0b6878f85a11496e3daMikulas Patocka _dm_io_cache = KMEM_CACHE(io, 0); 513952b355760c196ec014dd0b6878f85a11496e3daMikulas Patocka if (!_dm_io_cache) 514952b355760c196ec014dd0b6878f85a11496e3daMikulas Patocka return -ENOMEM; 515952b355760c196ec014dd0b6878f85a11496e3daMikulas Patocka 516952b355760c196ec014dd0b6878f85a11496e3daMikulas Patocka return 0; 517952b355760c196ec014dd0b6878f85a11496e3daMikulas Patocka} 518952b355760c196ec014dd0b6878f85a11496e3daMikulas Patocka 519952b355760c196ec014dd0b6878f85a11496e3daMikulas Patockavoid dm_io_exit(void) 520952b355760c196ec014dd0b6878f85a11496e3daMikulas Patocka{ 521952b355760c196ec014dd0b6878f85a11496e3daMikulas Patocka kmem_cache_destroy(_dm_io_cache); 522952b355760c196ec014dd0b6878f85a11496e3daMikulas Patocka _dm_io_cache = NULL; 523952b355760c196ec014dd0b6878f85a11496e3daMikulas Patocka} 524