1991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 2991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Copyright (C) 2011 Red Hat UK. 3991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 4991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * This file is released under the GPL. 5991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 6991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 7991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber#include "dm-thin-metadata.h" 8991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 9991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber#include <linux/device-mapper.h> 10991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber#include <linux/dm-io.h> 11991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber#include <linux/dm-kcopyd.h> 12991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber#include <linux/list.h> 13991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber#include <linux/init.h> 14991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber#include <linux/module.h> 15991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber#include <linux/slab.h> 16991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 17991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber#define DM_MSG_PREFIX "thin" 18991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 19991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 20991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Tunable constants 21991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 22991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber#define ENDIO_HOOK_POOL_SIZE 10240 23991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber#define DEFERRED_SET_SIZE 64 24991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber#define MAPPING_POOL_SIZE 1024 25991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber#define PRISON_CELLS 1024 26905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber#define COMMIT_PERIOD HZ 27991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 28991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 29991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * The block size of the device holding pool data must be 30991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * between 64KB and 1GB. 31991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 32991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber#define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (64 * 1024 >> SECTOR_SHIFT) 33991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber#define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT) 34991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 35991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 36991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Device id is restricted to 24 bits. 37991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 38991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber#define MAX_DEV_ID ((1 << 24) - 1) 39991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 40991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 41991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * How do we handle breaking sharing of data blocks? 42991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * ================================================= 43991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 44991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * We use a standard copy-on-write btree to store the mappings for the 45991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * devices (note I'm talking about copy-on-write of the metadata here, not 46991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * the data). When you take an internal snapshot you clone the root node 47991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * of the origin btree. After this there is no concept of an origin or a 48991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * snapshot. They are just two device trees that happen to point to the 49991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * same data blocks. 50991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 51991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * When we get a write in we decide if it's to a shared data block using 52991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * some timestamp magic. If it is, we have to break sharing. 53991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 54991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Let's say we write to a shared block in what was the origin. The 55991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * steps are: 56991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 57991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * i) plug io further to this physical block. (see bio_prison code). 58991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 59991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * ii) quiesce any read io to that shared data block. Obviously 60991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * including all devices that share this block. (see deferred_set code) 61991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 62991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * iii) copy the data block to a newly allocate block. This step can be 63991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * missed out if the io covers the block. (schedule_copy). 64991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 65991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * iv) insert the new mapping into the origin's btree 66fe878f34df89ad4af758f40bbec829807dc93a00Joe Thornber * (process_prepared_mapping). This act of inserting breaks some 67991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * sharing of btree nodes between the two devices. Breaking sharing only 68991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * effects the btree of that specific device. Btrees for the other 69991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * devices that share the block never change. The btree for the origin 70991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * device as it was after the last commit is untouched, ie. we're using 71991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * persistent data structures in the functional programming sense. 72991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 73991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * v) unplug io to this physical block, including the io that triggered 74991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * the breaking of sharing. 75991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 76991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Steps (ii) and (iii) occur in parallel. 77991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 78991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * The metadata _doesn't_ need to be committed before the io continues. We 79991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * get away with this because the io is always written to a _new_ block. 80991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * If there's a crash, then: 81991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 82991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * - The origin mapping will point to the old origin block (the shared 83991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * one). This will contain the data as it was before the io that triggered 84991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * the breaking of sharing came in. 85991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 86991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * - The snap mapping still points to the old block. As it would after 87991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * the commit. 88991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 89991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * The downside of this scheme is the timestamp magic isn't perfect, and 90991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * will continue to think that data block in the snapshot device is shared 91991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * even after the write to the origin has broken sharing. I suspect data 92991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * blocks will typically be shared by many different devices, so we're 93991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * breaking sharing n + 1 times, rather than n, where n is the number of 94991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * devices that reference this data block. At the moment I think the 95991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * benefits far, far outweigh the disadvantages. 96991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 97991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 98991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/*----------------------------------------------------------------*/ 99991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 100991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 101991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Sometimes we can't deal with a bio straight away. We put them in prison 102991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * where they can't cause any mischief. Bios are put in a cell identified 103991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * by a key, multiple bios can be in the same cell. When the cell is 104991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * subsequently unlocked the bios become available. 105991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 106991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstruct bio_prison; 107991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 108991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstruct cell_key { 109991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int virtual; 110991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_thin_id dev; 111991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t block; 112991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber}; 113991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 114991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstruct cell { 115991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct hlist_node list; 116991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio_prison *prison; 117991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct cell_key key; 1186f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber struct bio *holder; 119991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio_list bios; 120991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber}; 121991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 122991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstruct bio_prison { 123991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spinlock_t lock; 124991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mempool_t *cell_pool; 125991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 126991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned nr_buckets; 127991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned hash_mask; 128991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct hlist_head *cells; 129991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber}; 130991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 131991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic uint32_t calc_nr_buckets(unsigned nr_cells) 132991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 133991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber uint32_t n = 128; 134991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 135991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber nr_cells /= 4; 136991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber nr_cells = min(nr_cells, 8192u); 137991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 138991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber while (n < nr_cells) 139991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber n <<= 1; 140991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 141991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return n; 142991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 143991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 144991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 145991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * @nr_cells should be the number of cells you want in use _concurrently_. 146991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Don't confuse it with the number of distinct keys. 147991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 148991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic struct bio_prison *prison_create(unsigned nr_cells) 149991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 150991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned i; 151991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber uint32_t nr_buckets = calc_nr_buckets(nr_cells); 152991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber size_t len = sizeof(struct bio_prison) + 153991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber (sizeof(struct hlist_head) * nr_buckets); 154991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio_prison *prison = kmalloc(len, GFP_KERNEL); 155991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 156991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!prison) 157991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return NULL; 158991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 159991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_init(&prison->lock); 160991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber prison->cell_pool = mempool_create_kmalloc_pool(nr_cells, 161991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber sizeof(struct cell)); 162991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!prison->cell_pool) { 163991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber kfree(prison); 164991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return NULL; 165991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 166991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 167991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber prison->nr_buckets = nr_buckets; 168991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber prison->hash_mask = nr_buckets - 1; 169991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber prison->cells = (struct hlist_head *) (prison + 1); 170991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber for (i = 0; i < nr_buckets; i++) 171991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber INIT_HLIST_HEAD(prison->cells + i); 172991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 173991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return prison; 174991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 175991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 176991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void prison_destroy(struct bio_prison *prison) 177991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 178991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mempool_destroy(prison->cell_pool); 179991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber kfree(prison); 180991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 181991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 182991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic uint32_t hash_key(struct bio_prison *prison, struct cell_key *key) 183991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 184991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber const unsigned long BIG_PRIME = 4294967291UL; 185991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber uint64_t hash = key->block * BIG_PRIME; 186991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 187991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return (uint32_t) (hash & prison->hash_mask); 188991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 189991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 190991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int keys_equal(struct cell_key *lhs, struct cell_key *rhs) 191991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 192991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return (lhs->virtual == rhs->virtual) && 193991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber (lhs->dev == rhs->dev) && 194991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber (lhs->block == rhs->block); 195991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 196991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 197991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic struct cell *__search_bucket(struct hlist_head *bucket, 198991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct cell_key *key) 199991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 200991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct cell *cell; 201991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct hlist_node *tmp; 202991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 203991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber hlist_for_each_entry(cell, tmp, bucket, list) 204991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (keys_equal(&cell->key, key)) 205991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return cell; 206991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 207991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return NULL; 208991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 209991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 210991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 211991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * This may block if a new cell needs allocating. You must ensure that 212991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * cells will be unlocked even if the calling thread is blocked. 213991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 2146f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber * Returns 1 if the cell was already held, 0 if @inmate is the new holder. 215991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 216991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int bio_detain(struct bio_prison *prison, struct cell_key *key, 217991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio *inmate, struct cell **ref) 218991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2196f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber int r = 1; 220991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long flags; 221991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber uint32_t hash = hash_key(prison, key); 2226f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber struct cell *cell, *cell2; 223991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 224991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber BUG_ON(hash > prison->nr_buckets); 225991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 226991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&prison->lock, flags); 227991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2286f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber cell = __search_bucket(prison->cells + hash, key); 2296f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber if (cell) { 2306f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber bio_list_add(&cell->bios, inmate); 2316f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber goto out; 232991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 233991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2346f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber /* 2356f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber * Allocate a new cell 2366f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber */ 237991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&prison->lock, flags); 2386f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber cell2 = mempool_alloc(prison->cell_pool, GFP_NOIO); 2396f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber spin_lock_irqsave(&prison->lock, flags); 240991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2416f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber /* 2426f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber * We've been unlocked, so we have to double check that 2436f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber * nobody else has inserted this cell in the meantime. 2446f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber */ 2456f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber cell = __search_bucket(prison->cells + hash, key); 2466f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber if (cell) { 247991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mempool_free(cell2, prison->cell_pool); 2486f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber bio_list_add(&cell->bios, inmate); 2496f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber goto out; 2506f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber } 2516f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber 2526f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber /* 2536f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber * Use new cell. 2546f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber */ 2556f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber cell = cell2; 2566f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber 2576f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber cell->prison = prison; 2586f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber memcpy(&cell->key, key, sizeof(cell->key)); 2596f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber cell->holder = inmate; 2606f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber bio_list_init(&cell->bios); 2616f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber hlist_add_head(&cell->list, prison->cells + hash); 2626f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber 2636f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber r = 0; 2646f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber 2656f94a4c45a6f744383f9f695dde019998db3df55Joe Thornberout: 2666f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber spin_unlock_irqrestore(&prison->lock, flags); 267991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 268991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber *ref = cell; 269991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 270991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 271991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 272991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 273991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 274991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * @inmates must have been initialised prior to this call 275991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 276991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void __cell_release(struct cell *cell, struct bio_list *inmates) 277991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 278991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio_prison *prison = cell->prison; 279991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 280991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber hlist_del(&cell->list); 281991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 28203aaae7cdc71bc306888440b1f569d463e917b6dMike Snitzer if (inmates) { 28303aaae7cdc71bc306888440b1f569d463e917b6dMike Snitzer bio_list_add(inmates, cell->holder); 28403aaae7cdc71bc306888440b1f569d463e917b6dMike Snitzer bio_list_merge(inmates, &cell->bios); 28503aaae7cdc71bc306888440b1f569d463e917b6dMike Snitzer } 286991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 287991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mempool_free(cell, prison->cell_pool); 288991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 289991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 290991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void cell_release(struct cell *cell, struct bio_list *bios) 291991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 292991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long flags; 293991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio_prison *prison = cell->prison; 294991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 295991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&prison->lock, flags); 296991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber __cell_release(cell, bios); 297991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&prison->lock, flags); 298991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 299991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 300991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 301991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * There are a couple of places where we put a bio into a cell briefly 302991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * before taking it out again. In these situations we know that no other 303991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * bio may be in the cell. This function releases the cell, and also does 304991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * a sanity check. 305991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 3066f94a4c45a6f744383f9f695dde019998db3df55Joe Thornberstatic void __cell_release_singleton(struct cell *cell, struct bio *bio) 3076f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber{ 3086f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber BUG_ON(cell->holder != bio); 3096f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber BUG_ON(!bio_list_empty(&cell->bios)); 31003aaae7cdc71bc306888440b1f569d463e917b6dMike Snitzer 31103aaae7cdc71bc306888440b1f569d463e917b6dMike Snitzer __cell_release(cell, NULL); 3126f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber} 3136f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber 314991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void cell_release_singleton(struct cell *cell, struct bio *bio) 315991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 316991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long flags; 3176f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber struct bio_prison *prison = cell->prison; 318991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 319991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&prison->lock, flags); 3206f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber __cell_release_singleton(cell, bio); 321991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&prison->lock, flags); 3226f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber} 3236f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber 3246f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber/* 3256f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber * Sometimes we don't want the holder, just the additional bios. 3266f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber */ 3276f94a4c45a6f744383f9f695dde019998db3df55Joe Thornberstatic void __cell_release_no_holder(struct cell *cell, struct bio_list *inmates) 3286f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber{ 3296f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber struct bio_prison *prison = cell->prison; 3306f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber 3316f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber hlist_del(&cell->list); 3326f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber bio_list_merge(inmates, &cell->bios); 3336f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber 3346f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber mempool_free(cell, prison->cell_pool); 3356f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber} 3366f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber 3376f94a4c45a6f744383f9f695dde019998db3df55Joe Thornberstatic void cell_release_no_holder(struct cell *cell, struct bio_list *inmates) 3386f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber{ 3396f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber unsigned long flags; 3406f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber struct bio_prison *prison = cell->prison; 341991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 3426f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber spin_lock_irqsave(&prison->lock, flags); 3436f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber __cell_release_no_holder(cell, inmates); 3446f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber spin_unlock_irqrestore(&prison->lock, flags); 345991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 346991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 347991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void cell_error(struct cell *cell) 348991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 349991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio_prison *prison = cell->prison; 350991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio_list bios; 351991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio *bio; 352991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long flags; 353991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 354991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_init(&bios); 355991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 356991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&prison->lock, flags); 357991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber __cell_release(cell, &bios); 358991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&prison->lock, flags); 359991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 360991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber while ((bio = bio_list_pop(&bios))) 361991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_io_error(bio); 362991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 363991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 364991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/*----------------------------------------------------------------*/ 365991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 366991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 367991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * We use the deferred set to keep track of pending reads to shared blocks. 368991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * We do this to ensure the new mapping caused by a write isn't performed 369991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * until these prior reads have completed. Otherwise the insertion of the 370991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * new mapping could free the old block that the read bios are mapped to. 371991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 372991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 373991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstruct deferred_set; 374991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstruct deferred_entry { 375991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct deferred_set *ds; 376991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned count; 377991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct list_head work_items; 378991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber}; 379991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 380991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstruct deferred_set { 381991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spinlock_t lock; 382991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned current_entry; 383991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned sweeper; 384991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct deferred_entry entries[DEFERRED_SET_SIZE]; 385991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber}; 386991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 387991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void ds_init(struct deferred_set *ds) 388991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 389991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int i; 390991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 391991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_init(&ds->lock); 392991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ds->current_entry = 0; 393991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ds->sweeper = 0; 394991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber for (i = 0; i < DEFERRED_SET_SIZE; i++) { 395991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ds->entries[i].ds = ds; 396991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ds->entries[i].count = 0; 397991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber INIT_LIST_HEAD(&ds->entries[i].work_items); 398991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 399991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 400991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 401991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic struct deferred_entry *ds_inc(struct deferred_set *ds) 402991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 403991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long flags; 404991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct deferred_entry *entry; 405991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 406991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&ds->lock, flags); 407991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber entry = ds->entries + ds->current_entry; 408991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber entry->count++; 409991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&ds->lock, flags); 410991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 411991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return entry; 412991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 413991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 414991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic unsigned ds_next(unsigned index) 415991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 416991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return (index + 1) % DEFERRED_SET_SIZE; 417991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 418991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 419991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void __sweep(struct deferred_set *ds, struct list_head *head) 420991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 421991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber while ((ds->sweeper != ds->current_entry) && 422991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber !ds->entries[ds->sweeper].count) { 423991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber list_splice_init(&ds->entries[ds->sweeper].work_items, head); 424991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ds->sweeper = ds_next(ds->sweeper); 425991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 426991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 427991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if ((ds->sweeper == ds->current_entry) && !ds->entries[ds->sweeper].count) 428991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber list_splice_init(&ds->entries[ds->sweeper].work_items, head); 429991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 430991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 431991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void ds_dec(struct deferred_entry *entry, struct list_head *head) 432991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 433991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long flags; 434991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 435991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&entry->ds->lock, flags); 436991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber BUG_ON(!entry->count); 437991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber --entry->count; 438991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber __sweep(entry->ds, head); 439991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&entry->ds->lock, flags); 440991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 441991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 442991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 443991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Returns 1 if deferred or 0 if no pending items to delay job. 444991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 445991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int ds_add_work(struct deferred_set *ds, struct list_head *work) 446991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 447991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r = 1; 448991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long flags; 449991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned next_entry; 450991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 451991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&ds->lock, flags); 452991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if ((ds->sweeper == ds->current_entry) && 453991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber !ds->entries[ds->current_entry].count) 454991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = 0; 455991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber else { 456991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber list_add(work, &ds->entries[ds->current_entry].work_items); 457991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber next_entry = ds_next(ds->current_entry); 458991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!ds->entries[next_entry].count) 459991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ds->current_entry = next_entry; 460991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 461991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&ds->lock, flags); 462991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 463991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 464991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 465991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 466991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/*----------------------------------------------------------------*/ 467991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 468991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 469991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Key building. 470991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 471991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void build_data_key(struct dm_thin_device *td, 472991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t b, struct cell_key *key) 473991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 474991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber key->virtual = 0; 475991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber key->dev = dm_thin_dev_id(td); 476991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber key->block = b; 477991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 478991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 479991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void build_virtual_key(struct dm_thin_device *td, dm_block_t b, 480991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct cell_key *key) 481991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 482991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber key->virtual = 1; 483991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber key->dev = dm_thin_dev_id(td); 484991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber key->block = b; 485991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 486991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 487991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/*----------------------------------------------------------------*/ 488991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 489991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 490991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * A pool device ties together a metadata device and a data device. It 491991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * also provides the interface for creating and destroying internal 492991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * devices. 493991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 494991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstruct new_mapping; 49567e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber 49667e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornberstruct pool_features { 49767e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber unsigned zero_new_blocks:1; 49867e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber unsigned discard_enabled:1; 49967e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber unsigned discard_passdown:1; 50067e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber}; 50167e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber 502991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstruct pool { 503991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct list_head list; 504991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct dm_target *ti; /* Only set if a pool target is bound */ 505991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 506991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct mapped_device *pool_md; 507991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct block_device *md_dev; 508991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct dm_pool_metadata *pmd; 509991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 510991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber uint32_t sectors_per_block; 511991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned block_shift; 512991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t offset_mask; 513991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t low_water_blocks; 514991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 51567e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber struct pool_features pf; 516991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned low_water_triggered:1; /* A dm event has been sent */ 517991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned no_free_space:1; /* A -ENOSPC warning has been issued */ 518991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 519991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio_prison *prison; 520991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct dm_kcopyd_client *copier; 521991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 522991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct workqueue_struct *wq; 523991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct work_struct worker; 524905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber struct delayed_work waker; 525991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 526991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned ref_count; 527905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber unsigned long last_commit_jiffies; 528991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 529991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spinlock_t lock; 530991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio_list deferred_bios; 531991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio_list deferred_flush_bios; 532991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct list_head prepared_mappings; 533104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber struct list_head prepared_discards; 534991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 535991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio_list retry_on_resume_list; 536991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 537eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber struct deferred_set shared_read_ds; 538104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber struct deferred_set all_io_ds; 539991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 540991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct new_mapping *next_mapping; 541991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mempool_t *mapping_pool; 542991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mempool_t *endio_hook_pool; 543991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber}; 544991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 545991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 546991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Target context for a pool. 547991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 548991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstruct pool_c { 549991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct dm_target *ti; 550991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool; 551991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct dm_dev *data_dev; 552991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct dm_dev *metadata_dev; 553991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct dm_target_callbacks callbacks; 554991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 555991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t low_water_blocks; 55667e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber struct pool_features pf; 557991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber}; 558991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 559991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 560991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Target context for a thin. 561991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 562991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstruct thin_c { 563991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct dm_dev *pool_dev; 5642dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber struct dm_dev *origin_dev; 565991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_thin_id dev_id; 566991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 567991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool; 568991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct dm_thin_device *td; 569991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber}; 570991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 571991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/*----------------------------------------------------------------*/ 572991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 573991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 574991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * A global list of pools that uses a struct mapped_device as a key. 575991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 576991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic struct dm_thin_pool_table { 577991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct mutex mutex; 578991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct list_head pools; 579991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} dm_thin_pool_table; 580991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 581991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void pool_table_init(void) 582991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 583991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mutex_init(&dm_thin_pool_table.mutex); 584991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber INIT_LIST_HEAD(&dm_thin_pool_table.pools); 585991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 586991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 587991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void __pool_table_insert(struct pool *pool) 588991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 589991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 590991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber list_add(&pool->list, &dm_thin_pool_table.pools); 591991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 592991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 593991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void __pool_table_remove(struct pool *pool) 594991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 595991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 596991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber list_del(&pool->list); 597991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 598991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 599991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic struct pool *__pool_table_lookup(struct mapped_device *md) 600991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 601991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = NULL, *tmp; 602991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 603991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 604991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 605991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) { 606991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (tmp->pool_md == md) { 607991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool = tmp; 608991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber break; 609991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 610991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 611991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 612991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return pool; 613991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 614991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 615991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic struct pool *__pool_table_lookup_metadata_dev(struct block_device *md_dev) 616991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 617991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = NULL, *tmp; 618991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 619991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 620991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 621991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) { 622991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (tmp->md_dev == md_dev) { 623991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool = tmp; 624991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber break; 625991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 626991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 627991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 628991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return pool; 629991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 630991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 631991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/*----------------------------------------------------------------*/ 632991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 633eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornberstruct endio_hook { 634eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber struct thin_c *tc; 635eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber struct deferred_entry *shared_read_entry; 636104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber struct deferred_entry *all_io_entry; 637eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber struct new_mapping *overwrite_mapping; 638eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber}; 639eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber 640991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void __requeue_bio_list(struct thin_c *tc, struct bio_list *master) 641991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 642991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio *bio; 643991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio_list bios; 644991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 645991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_init(&bios); 646991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_merge(&bios, master); 647991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_init(master); 648991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 649991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber while ((bio = bio_list_pop(&bios))) { 650eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber struct endio_hook *h = dm_get_mapinfo(bio)->ptr; 651eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber if (h->tc == tc) 652991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_endio(bio, DM_ENDIO_REQUEUE); 653991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber else 654991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_add(master, bio); 655991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 656991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 657991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 658991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void requeue_io(struct thin_c *tc) 659991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 660991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = tc->pool; 661991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long flags; 662991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 663991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&pool->lock, flags); 664991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber __requeue_bio_list(tc, &pool->deferred_bios); 665991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber __requeue_bio_list(tc, &pool->retry_on_resume_list); 666991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&pool->lock, flags); 667991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 668991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 669991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 670991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * This section of code contains the logic for processing a thin device's IO. 671991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Much of the code depends on pool object resources (lists, workqueues, etc) 672991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * but most is exclusively called from the thin target rather than the thin-pool 673991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * target. 674991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 675991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 676991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic dm_block_t get_bio_block(struct thin_c *tc, struct bio *bio) 677991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 678991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return bio->bi_sector >> tc->pool->block_shift; 679991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 680991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 681991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void remap(struct thin_c *tc, struct bio *bio, dm_block_t block) 682991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 683991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = tc->pool; 684991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 685991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio->bi_bdev = tc->pool_dev->bdev; 686991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio->bi_sector = (block << pool->block_shift) + 687991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber (bio->bi_sector & pool->offset_mask); 688991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 689991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 6902dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornberstatic void remap_to_origin(struct thin_c *tc, struct bio *bio) 6912dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber{ 6922dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber bio->bi_bdev = tc->origin_dev->bdev; 6932dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber} 6942dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber 6952dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornberstatic void issue(struct thin_c *tc, struct bio *bio) 696991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 697991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = tc->pool; 698991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long flags; 699991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 700991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 701991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Batch together any FUA/FLUSH bios we find and then issue 702991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * a single commit for them in process_deferred_bios(). 703991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 704991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) { 705991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&pool->lock, flags); 706991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_add(&pool->deferred_flush_bios, bio); 707991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&pool->lock, flags); 708991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } else 709991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber generic_make_request(bio); 710991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 711991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 7122dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornberstatic void remap_to_origin_and_issue(struct thin_c *tc, struct bio *bio) 7132dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber{ 7142dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber remap_to_origin(tc, bio); 7152dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber issue(tc, bio); 7162dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber} 7172dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber 7182dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornberstatic void remap_and_issue(struct thin_c *tc, struct bio *bio, 7192dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber dm_block_t block) 7202dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber{ 7212dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber remap(tc, bio, block); 7222dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber issue(tc, bio); 7232dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber} 7242dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber 725991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 726991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * wake_worker() is used when new work is queued and when pool_resume is 727991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * ready to continue deferred IO processing. 728991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 729991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void wake_worker(struct pool *pool) 730991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 731991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber queue_work(pool->wq, &pool->worker); 732991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 733991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 734991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/*----------------------------------------------------------------*/ 735991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 736991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 737991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Bio endio functions. 738991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 739991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstruct new_mapping { 740991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct list_head list; 741991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 742eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber unsigned quiesced:1; 743eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber unsigned prepared:1; 744104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber unsigned pass_discard:1; 745991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 746991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct thin_c *tc; 747991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t virt_block; 748991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t data_block; 749104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber struct cell *cell, *cell2; 750991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int err; 751991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 752991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 753991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * If the bio covers the whole area of a block then we can avoid 754991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * zeroing or copying. Instead this bio is hooked. The bio will 755991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * still be in the cell, so care has to be taken to avoid issuing 756991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * the bio twice. 757991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 758991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio *bio; 759991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_end_io_t *saved_bi_end_io; 760991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber}; 761991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 762991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void __maybe_add_mapping(struct new_mapping *m) 763991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 764991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = m->tc->pool; 765991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 766eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber if (m->quiesced && m->prepared) { 767991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber list_add(&m->list, &pool->prepared_mappings); 768991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber wake_worker(pool); 769991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 770991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 771991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 772991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void copy_complete(int read_err, unsigned long write_err, void *context) 773991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 774991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long flags; 775991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct new_mapping *m = context; 776991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = m->tc->pool; 777991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 778991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber m->err = read_err || write_err ? -EIO : 0; 779991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 780991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&pool->lock, flags); 781991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber m->prepared = 1; 782991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber __maybe_add_mapping(m); 783991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&pool->lock, flags); 784991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 785991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 786991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void overwrite_endio(struct bio *bio, int err) 787991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 788991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long flags; 789eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber struct endio_hook *h = dm_get_mapinfo(bio)->ptr; 790eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber struct new_mapping *m = h->overwrite_mapping; 791991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = m->tc->pool; 792991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 793991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber m->err = err; 794991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 795991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&pool->lock, flags); 796991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber m->prepared = 1; 797991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber __maybe_add_mapping(m); 798991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&pool->lock, flags); 799991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 800991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 801991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/*----------------------------------------------------------------*/ 802991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 803991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 804991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Workqueue. 805991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 806991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 807991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 808991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Prepared mapping jobs. 809991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 810991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 811991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 812991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * This sends the bios in the cell back to the deferred_bios list. 813991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 814991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void cell_defer(struct thin_c *tc, struct cell *cell, 815991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t data_block) 816991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 817991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = tc->pool; 818991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long flags; 819991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 820991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&pool->lock, flags); 821991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber cell_release(cell, &pool->deferred_bios); 822991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&tc->pool->lock, flags); 823991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 824991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber wake_worker(pool); 825991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 826991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 827991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 828991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Same as cell_defer above, except it omits one particular detainee, 829991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * a write bio that covers the block and has already been processed. 830991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 8316f94a4c45a6f744383f9f695dde019998db3df55Joe Thornberstatic void cell_defer_except(struct thin_c *tc, struct cell *cell) 832991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 833991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio_list bios; 834991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = tc->pool; 835991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long flags; 836991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 837991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_init(&bios); 838991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 839991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&pool->lock, flags); 8406f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber cell_release_no_holder(cell, &pool->deferred_bios); 841991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&pool->lock, flags); 842991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 843991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber wake_worker(pool); 844991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 845991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 846991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void process_prepared_mapping(struct new_mapping *m) 847991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 848991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct thin_c *tc = m->tc; 849991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio *bio; 850991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 851991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 852991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio = m->bio; 853991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (bio) 854991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio->bi_end_io = m->saved_bi_end_io; 855991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 856991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (m->err) { 857991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber cell_error(m->cell); 858991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return; 859991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 860991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 861991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 862991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Commit the prepared block into the mapping btree. 863991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Any I/O for this block arriving after this point will get 864991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * remapped to it directly. 865991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 866991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_thin_insert_block(tc->td, m->virt_block, m->data_block); 867991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) { 868991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMERR("dm_thin_insert_block() failed"); 869991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber cell_error(m->cell); 870991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return; 871991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 872991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 873991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 874991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Release any bios held while the block was being provisioned. 875991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * If we are processing a write bio that completely covers the block, 876991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * we already processed it so can ignore it now when processing 877991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * the bios in the cell. 878991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 879991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (bio) { 8806f94a4c45a6f744383f9f695dde019998db3df55Joe Thornber cell_defer_except(tc, m->cell); 881991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_endio(bio, 0); 882991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } else 883991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber cell_defer(tc, m->cell, m->data_block); 884991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 885991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber list_del(&m->list); 886991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mempool_free(m, tc->pool->mapping_pool); 887991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 888991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 889104655fd4dcebd50068ef30253a001da72e3a081Joe Thornberstatic void process_prepared_discard(struct new_mapping *m) 890104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber{ 891104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber int r; 892104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber struct thin_c *tc = m->tc; 893104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber 894104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber r = dm_thin_remove_block(tc->td, m->virt_block); 895104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber if (r) 896104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber DMERR("dm_thin_remove_block() failed"); 897104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber 898104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber /* 899104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber * Pass the discard down to the underlying device? 900104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber */ 901104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber if (m->pass_discard) 902104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber remap_and_issue(tc, m->bio, m->data_block); 903104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber else 904104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber bio_endio(m->bio, 0); 905104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber 906104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber cell_defer_except(tc, m->cell); 907104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber cell_defer_except(tc, m->cell2); 908104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber mempool_free(m, tc->pool->mapping_pool); 909104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber} 910104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber 911104655fd4dcebd50068ef30253a001da72e3a081Joe Thornberstatic void process_prepared(struct pool *pool, struct list_head *head, 912104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber void (*fn)(struct new_mapping *)) 913991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 914991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long flags; 915991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct list_head maps; 916991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct new_mapping *m, *tmp; 917991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 918991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber INIT_LIST_HEAD(&maps); 919991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&pool->lock, flags); 920104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber list_splice_init(head, &maps); 921991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&pool->lock, flags); 922991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 923991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber list_for_each_entry_safe(m, tmp, &maps, list) 924104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber fn(m); 925991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 926991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 927991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 928991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Deferred bio jobs. 929991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 930104655fd4dcebd50068ef30253a001da72e3a081Joe Thornberstatic int io_overlaps_block(struct pool *pool, struct bio *bio) 931991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 932104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber return !(bio->bi_sector & pool->offset_mask) && 933991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber (bio->bi_size == (pool->sectors_per_block << SECTOR_SHIFT)); 934104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber 935104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber} 936104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber 937104655fd4dcebd50068ef30253a001da72e3a081Joe Thornberstatic int io_overwrites_block(struct pool *pool, struct bio *bio) 938104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber{ 939104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber return (bio_data_dir(bio) == WRITE) && 940104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber io_overlaps_block(pool, bio); 941991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 942991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 943991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void save_and_set_endio(struct bio *bio, bio_end_io_t **save, 944991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_end_io_t *fn) 945991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 946991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber *save = bio->bi_end_io; 947991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio->bi_end_io = fn; 948991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 949991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 950991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int ensure_next_mapping(struct pool *pool) 951991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 952991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (pool->next_mapping) 953991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return 0; 954991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 955991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->next_mapping = mempool_alloc(pool->mapping_pool, GFP_ATOMIC); 956991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 957991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return pool->next_mapping ? 0 : -ENOMEM; 958991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 959991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 960991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic struct new_mapping *get_next_mapping(struct pool *pool) 961991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 962991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct new_mapping *r = pool->next_mapping; 963991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 964991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber BUG_ON(!pool->next_mapping); 965991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 966991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->next_mapping = NULL; 967991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 968991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 969991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 970991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 971991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void schedule_copy(struct thin_c *tc, dm_block_t virt_block, 9722dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber struct dm_dev *origin, dm_block_t data_origin, 9732dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber dm_block_t data_dest, 974991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct cell *cell, struct bio *bio) 975991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 976991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 977991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = tc->pool; 978991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct new_mapping *m = get_next_mapping(pool); 979991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 980991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber INIT_LIST_HEAD(&m->list); 981eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber m->quiesced = 0; 982991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber m->prepared = 0; 983991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber m->tc = tc; 984991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber m->virt_block = virt_block; 985991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber m->data_block = data_dest; 986991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber m->cell = cell; 987991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber m->err = 0; 988991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber m->bio = NULL; 989991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 990eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber if (!ds_add_work(&pool->shared_read_ds, &m->list)) 991eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber m->quiesced = 1; 992991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 993991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 994991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * IO to pool_dev remaps to the pool target's data_dev. 995991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 996991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * If the whole block of data is being overwritten, we can issue the 997991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * bio immediately. Otherwise we use kcopyd to clone the data first. 998991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 999991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (io_overwrites_block(pool, bio)) { 1000eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber struct endio_hook *h = dm_get_mapinfo(bio)->ptr; 1001eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber h->overwrite_mapping = m; 1002991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber m->bio = bio; 1003991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio); 1004991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber remap_and_issue(tc, bio, data_dest); 1005991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } else { 1006991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct dm_io_region from, to; 1007991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 10082dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber from.bdev = origin->bdev; 1009991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber from.sector = data_origin * pool->sectors_per_block; 1010991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber from.count = pool->sectors_per_block; 1011991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1012991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber to.bdev = tc->pool_dev->bdev; 1013991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber to.sector = data_dest * pool->sectors_per_block; 1014991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber to.count = pool->sectors_per_block; 1015991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1016991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_kcopyd_copy(pool->copier, &from, 1, &to, 1017991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 0, copy_complete, m); 1018991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r < 0) { 1019991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mempool_free(m, pool->mapping_pool); 1020991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMERR("dm_kcopyd_copy() failed"); 1021991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber cell_error(cell); 1022991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1023991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1024991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1025991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 10262dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornberstatic void schedule_internal_copy(struct thin_c *tc, dm_block_t virt_block, 10272dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber dm_block_t data_origin, dm_block_t data_dest, 10282dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber struct cell *cell, struct bio *bio) 10292dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber{ 10302dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber schedule_copy(tc, virt_block, tc->pool_dev, 10312dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber data_origin, data_dest, cell, bio); 10322dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber} 10332dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber 10342dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornberstatic void schedule_external_copy(struct thin_c *tc, dm_block_t virt_block, 10352dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber dm_block_t data_dest, 10362dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber struct cell *cell, struct bio *bio) 10372dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber{ 10382dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber schedule_copy(tc, virt_block, tc->origin_dev, 10392dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber virt_block, data_dest, cell, bio); 10402dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber} 10412dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber 1042991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void schedule_zero(struct thin_c *tc, dm_block_t virt_block, 1043991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t data_block, struct cell *cell, 1044991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio *bio) 1045991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1046991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = tc->pool; 1047991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct new_mapping *m = get_next_mapping(pool); 1048991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1049991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber INIT_LIST_HEAD(&m->list); 1050eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber m->quiesced = 1; 1051991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber m->prepared = 0; 1052991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber m->tc = tc; 1053991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber m->virt_block = virt_block; 1054991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber m->data_block = data_block; 1055991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber m->cell = cell; 1056991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber m->err = 0; 1057991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber m->bio = NULL; 1058991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1059991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 1060991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * If the whole block of data is being overwritten or we are not 1061991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * zeroing pre-existing data, we can issue the bio immediately. 1062991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Otherwise we use kcopyd to zero the data first. 1063991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 106467e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber if (!pool->pf.zero_new_blocks) 1065991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber process_prepared_mapping(m); 1066991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1067991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber else if (io_overwrites_block(pool, bio)) { 1068eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber struct endio_hook *h = dm_get_mapinfo(bio)->ptr; 1069eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber h->overwrite_mapping = m; 1070991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber m->bio = bio; 1071991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio); 1072991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber remap_and_issue(tc, bio, data_block); 1073991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1074991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } else { 1075991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 1076991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct dm_io_region to; 1077991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1078991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber to.bdev = tc->pool_dev->bdev; 1079991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber to.sector = data_block * pool->sectors_per_block; 1080991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber to.count = pool->sectors_per_block; 1081991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1082991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_kcopyd_zero(pool->copier, 1, &to, 0, copy_complete, m); 1083991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r < 0) { 1084991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mempool_free(m, pool->mapping_pool); 1085991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMERR("dm_kcopyd_zero() failed"); 1086991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber cell_error(cell); 1087991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1088991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1089991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1090991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1091991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int alloc_data_block(struct thin_c *tc, dm_block_t *result) 1092991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1093991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 1094991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t free_blocks; 1095991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long flags; 1096991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = tc->pool; 1097991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1098991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_pool_get_free_block_count(pool->pmd, &free_blocks); 1099991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 1100991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 1101991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1102991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (free_blocks <= pool->low_water_blocks && !pool->low_water_triggered) { 1103991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMWARN("%s: reached low water mark, sending event.", 1104991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_device_name(pool->pool_md)); 1105991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&pool->lock, flags); 1106991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->low_water_triggered = 1; 1107991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&pool->lock, flags); 1108991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_table_event(pool->ti->table); 1109991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1110991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1111991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!free_blocks) { 1112991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (pool->no_free_space) 1113991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return -ENOSPC; 1114991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber else { 1115991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 1116991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Try to commit to see if that will free up some 1117991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * more space. 1118991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 1119991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_pool_commit_metadata(pool->pmd); 1120991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) { 1121991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMERR("%s: dm_pool_commit_metadata() failed, error = %d", 1122991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber __func__, r); 1123991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 1124991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1125991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1126991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_pool_get_free_block_count(pool->pmd, &free_blocks); 1127991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 1128991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 1129991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1130991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 1131991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * If we still have no space we set a flag to avoid 1132991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * doing all this checking and return -ENOSPC. 1133991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 1134991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!free_blocks) { 1135991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMWARN("%s: no free space available.", 1136991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_device_name(pool->pool_md)); 1137991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&pool->lock, flags); 1138991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->no_free_space = 1; 1139991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&pool->lock, flags); 1140991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return -ENOSPC; 1141991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1142991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1143991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1144991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1145991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_pool_alloc_data_block(pool->pmd, result); 1146991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 1147991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 1148991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1149991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return 0; 1150991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1151991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1152991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 1153991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * If we have run out of space, queue bios until the device is 1154991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * resumed, presumably after having been reloaded with more space. 1155991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 1156991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void retry_on_resume(struct bio *bio) 1157991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1158eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber struct endio_hook *h = dm_get_mapinfo(bio)->ptr; 1159eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber struct thin_c *tc = h->tc; 1160991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = tc->pool; 1161991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long flags; 1162991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1163991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&pool->lock, flags); 1164991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_add(&pool->retry_on_resume_list, bio); 1165991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&pool->lock, flags); 1166991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1167991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1168991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void no_space(struct cell *cell) 1169991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1170991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio *bio; 1171991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio_list bios; 1172991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1173991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_init(&bios); 1174991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber cell_release(cell, &bios); 1175991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1176991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber while ((bio = bio_list_pop(&bios))) 1177991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber retry_on_resume(bio); 1178991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1179991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1180104655fd4dcebd50068ef30253a001da72e3a081Joe Thornberstatic void process_discard(struct thin_c *tc, struct bio *bio) 1181104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber{ 1182104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber int r; 1183c3a0ce2eab76daf9516c817c3f227ea3f4549bd8Mike Snitzer unsigned long flags; 1184104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber struct pool *pool = tc->pool; 1185104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber struct cell *cell, *cell2; 1186104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber struct cell_key key, key2; 1187104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber dm_block_t block = get_bio_block(tc, bio); 1188104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber struct dm_thin_lookup_result lookup_result; 1189104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber struct new_mapping *m; 1190104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber 1191104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber build_virtual_key(tc->td, block, &key); 1192104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber if (bio_detain(tc->pool->prison, &key, bio, &cell)) 1193104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber return; 1194104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber 1195104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber r = dm_thin_find_block(tc->td, block, 1, &lookup_result); 1196104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber switch (r) { 1197104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber case 0: 1198104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber /* 1199104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber * Check nobody is fiddling with this pool block. This can 1200104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber * happen if someone's in the process of breaking sharing 1201104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber * on this block. 1202104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber */ 1203104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber build_data_key(tc->td, lookup_result.block, &key2); 1204104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber if (bio_detain(tc->pool->prison, &key2, bio, &cell2)) { 1205104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber cell_release_singleton(cell, bio); 1206104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber break; 1207104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber } 1208104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber 1209104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber if (io_overlaps_block(pool, bio)) { 1210104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber /* 1211104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber * IO may still be going to the destination block. We must 1212104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber * quiesce before we can do the removal. 1213104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber */ 1214104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber m = get_next_mapping(pool); 1215104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber m->tc = tc; 121667e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber m->pass_discard = (!lookup_result.shared) & pool->pf.discard_passdown; 1217104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber m->virt_block = block; 1218104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber m->data_block = lookup_result.block; 1219104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber m->cell = cell; 1220104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber m->cell2 = cell2; 1221104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber m->err = 0; 1222104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber m->bio = bio; 1223104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber 1224104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber if (!ds_add_work(&pool->all_io_ds, &m->list)) { 1225c3a0ce2eab76daf9516c817c3f227ea3f4549bd8Mike Snitzer spin_lock_irqsave(&pool->lock, flags); 1226104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber list_add(&m->list, &pool->prepared_discards); 1227c3a0ce2eab76daf9516c817c3f227ea3f4549bd8Mike Snitzer spin_unlock_irqrestore(&pool->lock, flags); 1228104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber wake_worker(pool); 1229104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber } 1230104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber } else { 1231104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber /* 1232104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber * This path is hit if people are ignoring 1233104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber * limits->discard_granularity. It ignores any 1234104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber * part of the discard that is in a subsequent 1235104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber * block. 1236104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber */ 1237104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber sector_t offset = bio->bi_sector - (block << pool->block_shift); 1238104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber unsigned remaining = (pool->sectors_per_block - offset) << 9; 1239104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber bio->bi_size = min(bio->bi_size, remaining); 1240104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber 1241104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber cell_release_singleton(cell, bio); 1242104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber cell_release_singleton(cell2, bio); 1243104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber remap_and_issue(tc, bio, lookup_result.block); 1244104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber } 1245104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber break; 1246104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber 1247104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber case -ENODATA: 1248104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber /* 1249104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber * It isn't provisioned, just forget it. 1250104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber */ 1251104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber cell_release_singleton(cell, bio); 1252104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber bio_endio(bio, 0); 1253104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber break; 1254104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber 1255104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber default: 1256104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber DMERR("discard: find block unexpectedly returned %d", r); 1257104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber cell_release_singleton(cell, bio); 1258104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber bio_io_error(bio); 1259104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber break; 1260104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber } 1261104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber} 1262104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber 1263991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void break_sharing(struct thin_c *tc, struct bio *bio, dm_block_t block, 1264991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct cell_key *key, 1265991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct dm_thin_lookup_result *lookup_result, 1266991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct cell *cell) 1267991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1268991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 1269991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t data_block; 1270991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1271991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = alloc_data_block(tc, &data_block); 1272991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber switch (r) { 1273991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber case 0: 12742dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber schedule_internal_copy(tc, block, lookup_result->block, 12752dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber data_block, cell, bio); 1276991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber break; 1277991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1278991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber case -ENOSPC: 1279991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber no_space(cell); 1280991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber break; 1281991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1282991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber default: 1283991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMERR("%s: alloc_data_block() failed, error = %d", __func__, r); 1284991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber cell_error(cell); 1285991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber break; 1286991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1287991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1288991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1289991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void process_shared_bio(struct thin_c *tc, struct bio *bio, 1290991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t block, 1291991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct dm_thin_lookup_result *lookup_result) 1292991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1293991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct cell *cell; 1294991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = tc->pool; 1295991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct cell_key key; 1296991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1297991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 1298991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * If cell is already occupied, then sharing is already in the process 1299991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * of being broken so we have nothing further to do here. 1300991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 1301991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber build_data_key(tc->td, lookup_result->block, &key); 1302991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (bio_detain(pool->prison, &key, bio, &cell)) 1303991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return; 1304991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1305991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (bio_data_dir(bio) == WRITE) 1306991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber break_sharing(tc, bio, block, &key, lookup_result, cell); 1307991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber else { 1308eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber struct endio_hook *h = dm_get_mapinfo(bio)->ptr; 1309991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1310eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber h->shared_read_entry = ds_inc(&pool->shared_read_ds); 1311991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1312991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber cell_release_singleton(cell, bio); 1313991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber remap_and_issue(tc, bio, lookup_result->block); 1314991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1315991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1316991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1317991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block, 1318991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct cell *cell) 1319991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1320991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 1321991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t data_block; 1322991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1323991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 1324991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Remap empty bios (flushes) immediately, without provisioning. 1325991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 1326991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!bio->bi_size) { 1327991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber cell_release_singleton(cell, bio); 1328991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber remap_and_issue(tc, bio, 0); 1329991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return; 1330991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1331991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1332991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 1333991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Fill read bios with zeroes and complete them immediately. 1334991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 1335991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (bio_data_dir(bio) == READ) { 1336991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber zero_fill_bio(bio); 1337991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber cell_release_singleton(cell, bio); 1338991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_endio(bio, 0); 1339991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return; 1340991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1341991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1342991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = alloc_data_block(tc, &data_block); 1343991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber switch (r) { 1344991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber case 0: 13452dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber if (tc->origin_dev) 13462dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber schedule_external_copy(tc, block, data_block, cell, bio); 13472dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber else 13482dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber schedule_zero(tc, block, data_block, cell, bio); 1349991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber break; 1350991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1351991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber case -ENOSPC: 1352991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber no_space(cell); 1353991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber break; 1354991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1355991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber default: 1356991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMERR("%s: alloc_data_block() failed, error = %d", __func__, r); 1357991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber cell_error(cell); 1358991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber break; 1359991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1360991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1361991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1362991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void process_bio(struct thin_c *tc, struct bio *bio) 1363991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1364991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 1365991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t block = get_bio_block(tc, bio); 1366991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct cell *cell; 1367991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct cell_key key; 1368991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct dm_thin_lookup_result lookup_result; 1369991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1370991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 1371991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * If cell is already occupied, then the block is already 1372991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * being provisioned so we have nothing further to do here. 1373991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 1374991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber build_virtual_key(tc->td, block, &key); 1375991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (bio_detain(tc->pool->prison, &key, bio, &cell)) 1376991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return; 1377991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1378991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_thin_find_block(tc->td, block, 1, &lookup_result); 1379991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber switch (r) { 1380991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber case 0: 1381991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 1382991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * We can release this cell now. This thread is the only 1383991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * one that puts bios into a cell, and we know there were 1384991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * no preceding bios. 1385991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 1386991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 1387991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * TODO: this will probably have to change when discard goes 1388991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * back in. 1389991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 1390991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber cell_release_singleton(cell, bio); 1391991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1392991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (lookup_result.shared) 1393991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber process_shared_bio(tc, bio, block, &lookup_result); 1394991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber else 1395991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber remap_and_issue(tc, bio, lookup_result.block); 1396991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber break; 1397991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1398991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber case -ENODATA: 13992dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber if (bio_data_dir(bio) == READ && tc->origin_dev) { 14002dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber cell_release_singleton(cell, bio); 14012dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber remap_to_origin_and_issue(tc, bio); 14022dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber } else 14032dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber provision_block(tc, bio, block, cell); 1404991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber break; 1405991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1406991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber default: 1407991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMERR("dm_thin_find_block() failed, error = %d", r); 1408104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber cell_release_singleton(cell, bio); 1409991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_io_error(bio); 1410991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber break; 1411991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1412991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1413991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1414905e51b39a5558706a6ed883fe104de3d417050bJoe Thornberstatic int need_commit_due_to_time(struct pool *pool) 1415905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber{ 1416905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber return jiffies < pool->last_commit_jiffies || 1417905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber jiffies > pool->last_commit_jiffies + COMMIT_PERIOD; 1418905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber} 1419905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber 1420991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void process_deferred_bios(struct pool *pool) 1421991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1422991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long flags; 1423991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio *bio; 1424991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio_list bios; 1425991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 1426991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1427991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_init(&bios); 1428991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1429991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&pool->lock, flags); 1430991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_merge(&bios, &pool->deferred_bios); 1431991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_init(&pool->deferred_bios); 1432991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&pool->lock, flags); 1433991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1434991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber while ((bio = bio_list_pop(&bios))) { 1435eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber struct endio_hook *h = dm_get_mapinfo(bio)->ptr; 1436eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber struct thin_c *tc = h->tc; 1437eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber 1438991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 1439991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * If we've got no free new_mapping structs, and processing 1440991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * this bio might require one, we pause until there are some 1441991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * prepared mappings to process. 1442991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 1443991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (ensure_next_mapping(pool)) { 1444991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&pool->lock, flags); 1445991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_merge(&pool->deferred_bios, &bios); 1446991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&pool->lock, flags); 1447991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1448991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber break; 1449991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1450104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber 1451104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber if (bio->bi_rw & REQ_DISCARD) 1452104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber process_discard(tc, bio); 1453104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber else 1454104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber process_bio(tc, bio); 1455991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1456991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1457991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 1458991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * If there are any deferred flush bios, we must commit 1459991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * the metadata before issuing them. 1460991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 1461991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_init(&bios); 1462991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&pool->lock, flags); 1463991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_merge(&bios, &pool->deferred_flush_bios); 1464991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_init(&pool->deferred_flush_bios); 1465991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&pool->lock, flags); 1466991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1467905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber if (bio_list_empty(&bios) && !need_commit_due_to_time(pool)) 1468991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return; 1469991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1470991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_pool_commit_metadata(pool->pmd); 1471991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) { 1472991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMERR("%s: dm_pool_commit_metadata() failed, error = %d", 1473991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber __func__, r); 1474991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber while ((bio = bio_list_pop(&bios))) 1475991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_io_error(bio); 1476991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return; 1477991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1478905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber pool->last_commit_jiffies = jiffies; 1479991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1480991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber while ((bio = bio_list_pop(&bios))) 1481991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber generic_make_request(bio); 1482991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1483991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1484991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void do_worker(struct work_struct *ws) 1485991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1486991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = container_of(ws, struct pool, worker); 1487991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1488104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber process_prepared(pool, &pool->prepared_mappings, process_prepared_mapping); 1489104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber process_prepared(pool, &pool->prepared_discards, process_prepared_discard); 1490991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber process_deferred_bios(pool); 1491991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1492991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1493905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber/* 1494905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber * We want to commit periodically so that not too much 1495905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber * unwritten data builds up. 1496905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber */ 1497905e51b39a5558706a6ed883fe104de3d417050bJoe Thornberstatic void do_waker(struct work_struct *ws) 1498905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber{ 1499905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber struct pool *pool = container_of(to_delayed_work(ws), struct pool, waker); 1500905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber wake_worker(pool); 1501905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber queue_delayed_work(pool->wq, &pool->waker, COMMIT_PERIOD); 1502905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber} 1503905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber 1504991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/*----------------------------------------------------------------*/ 1505991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1506991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 1507991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Mapping functions. 1508991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 1509991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1510991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 1511991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Called only while mapping a thin bio to hand it over to the workqueue. 1512991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 1513991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void thin_defer_bio(struct thin_c *tc, struct bio *bio) 1514991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1515991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long flags; 1516991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = tc->pool; 1517991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1518991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&pool->lock, flags); 1519991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_add(&pool->deferred_bios, bio); 1520991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&pool->lock, flags); 1521991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1522991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber wake_worker(pool); 1523991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1524991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1525eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornberstatic struct endio_hook *thin_hook_bio(struct thin_c *tc, struct bio *bio) 1526eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber{ 1527eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber struct pool *pool = tc->pool; 1528eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber struct endio_hook *h = mempool_alloc(pool->endio_hook_pool, GFP_NOIO); 1529eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber 1530eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber h->tc = tc; 1531eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber h->shared_read_entry = NULL; 1532104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber h->all_io_entry = bio->bi_rw & REQ_DISCARD ? NULL : ds_inc(&pool->all_io_ds); 1533eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber h->overwrite_mapping = NULL; 1534eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber 1535eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber return h; 1536eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber} 1537eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber 1538991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 1539991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Non-blocking function called from the thin target's map function. 1540991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 1541991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int thin_bio_map(struct dm_target *ti, struct bio *bio, 1542991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber union map_info *map_context) 1543991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1544991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 1545991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct thin_c *tc = ti->private; 1546991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t block = get_bio_block(tc, bio); 1547991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct dm_thin_device *td = tc->td; 1548991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct dm_thin_lookup_result result; 1549991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1550eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber map_context->ptr = thin_hook_bio(tc, bio); 1551104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber if (bio->bi_rw & (REQ_DISCARD | REQ_FLUSH | REQ_FUA)) { 1552991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber thin_defer_bio(tc, bio); 1553991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return DM_MAPIO_SUBMITTED; 1554991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1555991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1556991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_thin_find_block(td, block, 0, &result); 1557991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1558991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 1559991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Note that we defer readahead too. 1560991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 1561991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber switch (r) { 1562991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber case 0: 1563991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (unlikely(result.shared)) { 1564991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 1565991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * We have a race condition here between the 1566991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * result.shared value returned by the lookup and 1567991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * snapshot creation, which may cause new 1568991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * sharing. 1569991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 1570991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * To avoid this always quiesce the origin before 1571991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * taking the snap. You want to do this anyway to 1572991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * ensure a consistent application view 1573991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * (i.e. lockfs). 1574991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 1575991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * More distant ancestors are irrelevant. The 1576991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * shared flag will be set in their case. 1577991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 1578991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber thin_defer_bio(tc, bio); 1579991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = DM_MAPIO_SUBMITTED; 1580991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } else { 1581991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber remap(tc, bio, result.block); 1582991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = DM_MAPIO_REMAPPED; 1583991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1584991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber break; 1585991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1586991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber case -ENODATA: 1587991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 1588991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * In future, the failed dm_thin_find_block above could 1589991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * provide the hint to load the metadata into cache. 1590991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 1591991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber case -EWOULDBLOCK: 1592991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber thin_defer_bio(tc, bio); 1593991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = DM_MAPIO_SUBMITTED; 1594991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber break; 1595991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1596991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1597991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 1598991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1599991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1600991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int pool_is_congested(struct dm_target_callbacks *cb, int bdi_bits) 1601991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1602991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 1603991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long flags; 1604991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool_c *pt = container_of(cb, struct pool_c, callbacks); 1605991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1606991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&pt->pool->lock, flags); 1607991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = !bio_list_empty(&pt->pool->retry_on_resume_list); 1608991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&pt->pool->lock, flags); 1609991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1610991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!r) { 1611991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct request_queue *q = bdev_get_queue(pt->data_dev->bdev); 1612991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = bdi_congested(&q->backing_dev_info, bdi_bits); 1613991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1614991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1615991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 1616991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1617991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1618991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void __requeue_bios(struct pool *pool) 1619991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1620991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_merge(&pool->deferred_bios, &pool->retry_on_resume_list); 1621991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_init(&pool->retry_on_resume_list); 1622991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1623991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1624991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/*---------------------------------------------------------------- 1625991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Binding of control targets to a pool object 1626991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber *--------------------------------------------------------------*/ 1627991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int bind_control_target(struct pool *pool, struct dm_target *ti) 1628991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1629991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool_c *pt = ti->private; 1630991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1631991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->ti = ti; 1632991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->low_water_blocks = pt->low_water_blocks; 163367e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber pool->pf = pt->pf; 1634991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1635f402693d06f32e746c6153e459c5fb064fa02741Mike Snitzer /* 1636f402693d06f32e746c6153e459c5fb064fa02741Mike Snitzer * If discard_passdown was enabled verify that the data device 1637f402693d06f32e746c6153e459c5fb064fa02741Mike Snitzer * supports discards. Disable discard_passdown if not; otherwise 1638f402693d06f32e746c6153e459c5fb064fa02741Mike Snitzer * -EOPNOTSUPP will be returned. 1639f402693d06f32e746c6153e459c5fb064fa02741Mike Snitzer */ 1640f402693d06f32e746c6153e459c5fb064fa02741Mike Snitzer if (pt->pf.discard_passdown) { 1641f402693d06f32e746c6153e459c5fb064fa02741Mike Snitzer struct request_queue *q = bdev_get_queue(pt->data_dev->bdev); 1642f402693d06f32e746c6153e459c5fb064fa02741Mike Snitzer if (!q || !blk_queue_discard(q)) { 1643f402693d06f32e746c6153e459c5fb064fa02741Mike Snitzer char buf[BDEVNAME_SIZE]; 1644f402693d06f32e746c6153e459c5fb064fa02741Mike Snitzer DMWARN("Discard unsupported by data device (%s): Disabling discard passdown.", 1645f402693d06f32e746c6153e459c5fb064fa02741Mike Snitzer bdevname(pt->data_dev->bdev, buf)); 1646f402693d06f32e746c6153e459c5fb064fa02741Mike Snitzer pool->pf.discard_passdown = 0; 1647f402693d06f32e746c6153e459c5fb064fa02741Mike Snitzer } 1648f402693d06f32e746c6153e459c5fb064fa02741Mike Snitzer } 1649f402693d06f32e746c6153e459c5fb064fa02741Mike Snitzer 1650991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return 0; 1651991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1652991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1653991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void unbind_control_target(struct pool *pool, struct dm_target *ti) 1654991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1655991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (pool->ti == ti) 1656991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->ti = NULL; 1657991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1658991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1659991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/*---------------------------------------------------------------- 1660991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Pool creation 1661991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber *--------------------------------------------------------------*/ 166267e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber/* Initialize pool features. */ 166367e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornberstatic void pool_features_init(struct pool_features *pf) 166467e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber{ 166567e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber pf->zero_new_blocks = 1; 166667e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber pf->discard_enabled = 1; 166767e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber pf->discard_passdown = 1; 166867e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber} 166967e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber 1670991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void __pool_destroy(struct pool *pool) 1671991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1672991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber __pool_table_remove(pool); 1673991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1674991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (dm_pool_metadata_close(pool->pmd) < 0) 1675991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMWARN("%s: dm_pool_metadata_close() failed.", __func__); 1676991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1677991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber prison_destroy(pool->prison); 1678991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_kcopyd_client_destroy(pool->copier); 1679991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1680991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (pool->wq) 1681991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber destroy_workqueue(pool->wq); 1682991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1683991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (pool->next_mapping) 1684991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mempool_free(pool->next_mapping, pool->mapping_pool); 1685991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mempool_destroy(pool->mapping_pool); 1686991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mempool_destroy(pool->endio_hook_pool); 1687991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber kfree(pool); 1688991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1689991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1690991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic struct pool *pool_create(struct mapped_device *pool_md, 1691991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct block_device *metadata_dev, 1692991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long block_size, char **error) 1693991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1694991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 1695991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber void *err_p; 1696991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool; 1697991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct dm_pool_metadata *pmd; 1698991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1699991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pmd = dm_pool_metadata_open(metadata_dev, block_size); 1700991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (IS_ERR(pmd)) { 1701991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber *error = "Error creating metadata object"; 1702991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return (struct pool *)pmd; 1703991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1704991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1705991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool = kmalloc(sizeof(*pool), GFP_KERNEL); 1706991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!pool) { 1707991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber *error = "Error allocating memory for pool"; 1708991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber err_p = ERR_PTR(-ENOMEM); 1709991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber goto bad_pool; 1710991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1711991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1712991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->pmd = pmd; 1713991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->sectors_per_block = block_size; 1714991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->block_shift = ffs(block_size) - 1; 1715991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->offset_mask = block_size - 1; 1716991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->low_water_blocks = 0; 171767e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber pool_features_init(&pool->pf); 1718991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->prison = prison_create(PRISON_CELLS); 1719991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!pool->prison) { 1720991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber *error = "Error creating pool's bio prison"; 1721991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber err_p = ERR_PTR(-ENOMEM); 1722991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber goto bad_prison; 1723991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1724991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1725991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->copier = dm_kcopyd_client_create(); 1726991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (IS_ERR(pool->copier)) { 1727991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = PTR_ERR(pool->copier); 1728991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber *error = "Error creating pool's kcopyd client"; 1729991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber err_p = ERR_PTR(r); 1730991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber goto bad_kcopyd_client; 1731991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1732991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1733991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 1734991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Create singlethreaded workqueue that will service all devices 1735991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * that use this metadata. 1736991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 1737991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM); 1738991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!pool->wq) { 1739991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber *error = "Error creating pool's workqueue"; 1740991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber err_p = ERR_PTR(-ENOMEM); 1741991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber goto bad_wq; 1742991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1743991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1744991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber INIT_WORK(&pool->worker, do_worker); 1745905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber INIT_DELAYED_WORK(&pool->waker, do_waker); 1746991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_init(&pool->lock); 1747991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_init(&pool->deferred_bios); 1748991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_init(&pool->deferred_flush_bios); 1749991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber INIT_LIST_HEAD(&pool->prepared_mappings); 1750104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber INIT_LIST_HEAD(&pool->prepared_discards); 1751991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->low_water_triggered = 0; 1752991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->no_free_space = 0; 1753991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio_list_init(&pool->retry_on_resume_list); 1754eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber ds_init(&pool->shared_read_ds); 1755104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber ds_init(&pool->all_io_ds); 1756991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1757991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->next_mapping = NULL; 1758991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->mapping_pool = 1759991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mempool_create_kmalloc_pool(MAPPING_POOL_SIZE, sizeof(struct new_mapping)); 1760991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!pool->mapping_pool) { 1761991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber *error = "Error creating pool's mapping mempool"; 1762991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber err_p = ERR_PTR(-ENOMEM); 1763991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber goto bad_mapping_pool; 1764991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1765991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1766991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->endio_hook_pool = 1767991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mempool_create_kmalloc_pool(ENDIO_HOOK_POOL_SIZE, sizeof(struct endio_hook)); 1768991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!pool->endio_hook_pool) { 1769991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber *error = "Error creating pool's endio_hook mempool"; 1770991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber err_p = ERR_PTR(-ENOMEM); 1771991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber goto bad_endio_hook_pool; 1772991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1773991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->ref_count = 1; 1774905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber pool->last_commit_jiffies = jiffies; 1775991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->pool_md = pool_md; 1776991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->md_dev = metadata_dev; 1777991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber __pool_table_insert(pool); 1778991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1779991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return pool; 1780991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1781991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberbad_endio_hook_pool: 1782991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mempool_destroy(pool->mapping_pool); 1783991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberbad_mapping_pool: 1784991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber destroy_workqueue(pool->wq); 1785991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberbad_wq: 1786991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_kcopyd_client_destroy(pool->copier); 1787991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberbad_kcopyd_client: 1788991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber prison_destroy(pool->prison); 1789991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberbad_prison: 1790991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber kfree(pool); 1791991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberbad_pool: 1792991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (dm_pool_metadata_close(pmd)) 1793991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMWARN("%s: dm_pool_metadata_close() failed.", __func__); 1794991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1795991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return err_p; 1796991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1797991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1798991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void __pool_inc(struct pool *pool) 1799991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1800991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 1801991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->ref_count++; 1802991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1803991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1804991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void __pool_dec(struct pool *pool) 1805991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1806991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 1807991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber BUG_ON(!pool->ref_count); 1808991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!--pool->ref_count) 1809991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber __pool_destroy(pool); 1810991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1811991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1812991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic struct pool *__pool_find(struct mapped_device *pool_md, 1813991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct block_device *metadata_dev, 181467e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber unsigned long block_size, char **error, 181567e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber int *created) 1816991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1817991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = __pool_table_lookup_metadata_dev(metadata_dev); 1818991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1819991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (pool) { 1820991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (pool->pool_md != pool_md) 1821991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return ERR_PTR(-EBUSY); 1822991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber __pool_inc(pool); 1823991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1824991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } else { 1825991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool = __pool_table_lookup(pool_md); 1826991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (pool) { 1827991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (pool->md_dev != metadata_dev) 1828991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return ERR_PTR(-EINVAL); 1829991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber __pool_inc(pool); 1830991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 183167e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber } else { 1832991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool = pool_create(pool_md, metadata_dev, block_size, error); 183367e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber *created = 1; 183467e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber } 1835991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1836991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1837991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return pool; 1838991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1839991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1840991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/*---------------------------------------------------------------- 1841991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Pool target methods 1842991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber *--------------------------------------------------------------*/ 1843991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void pool_dtr(struct dm_target *ti) 1844991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1845991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool_c *pt = ti->private; 1846991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1847991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mutex_lock(&dm_thin_pool_table.mutex); 1848991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1849991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unbind_control_target(pt->pool, ti); 1850991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber __pool_dec(pt->pool); 1851991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_put_device(ti, pt->metadata_dev); 1852991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_put_device(ti, pt->data_dev); 1853991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber kfree(pt); 1854991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1855991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mutex_unlock(&dm_thin_pool_table.mutex); 1856991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1857991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1858991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int parse_pool_features(struct dm_arg_set *as, struct pool_features *pf, 1859991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct dm_target *ti) 1860991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 1861991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 1862991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned argc; 1863991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber const char *arg_name; 1864991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1865991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber static struct dm_arg _args[] = { 186667e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber {0, 3, "Invalid number of pool feature arguments"}, 1867991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber }; 1868991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1869991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 1870991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * No feature arguments supplied. 1871991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 1872991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!as->argc) 1873991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return 0; 1874991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1875991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_read_arg_group(_args, as, &argc, &ti->error); 1876991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 1877991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return -EINVAL; 1878991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1879991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber while (argc && !r) { 1880991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber arg_name = dm_shift_arg(as); 1881991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber argc--; 1882991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1883991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!strcasecmp(arg_name, "skip_block_zeroing")) { 1884991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pf->zero_new_blocks = 0; 1885991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber continue; 188667e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber } else if (!strcasecmp(arg_name, "ignore_discard")) { 188767e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber pf->discard_enabled = 0; 188867e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber continue; 188967e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber } else if (!strcasecmp(arg_name, "no_discard_passdown")) { 189067e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber pf->discard_passdown = 0; 189167e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber continue; 1892991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1893991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1894991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ti->error = "Unrecognised pool feature requested"; 1895991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = -EINVAL; 1896991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1897991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1898991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 1899991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 1900991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1901991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 1902991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * thin-pool <metadata dev> <data dev> 1903991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * <data block size (sectors)> 1904991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * <low water mark (blocks)> 1905991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * [<#feature args> [<arg>]*] 1906991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 1907991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Optional feature arguments are: 1908991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * skip_block_zeroing: skips the zeroing of newly-provisioned blocks. 190967e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber * ignore_discard: disable discard 191067e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber * no_discard_passdown: don't pass discards down to the data device 1911991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 1912991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int pool_ctr(struct dm_target *ti, unsigned argc, char **argv) 1913991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 191467e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber int r, pool_created = 0; 1915991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool_c *pt; 1916991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool; 1917991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool_features pf; 1918991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct dm_arg_set as; 1919991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct dm_dev *data_dev; 1920991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long block_size; 1921991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t low_water_blocks; 1922991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct dm_dev *metadata_dev; 1923991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber sector_t metadata_dev_size; 1924c4a69ecdb463a901b4645230613961e134e897cdMike Snitzer char b[BDEVNAME_SIZE]; 1925991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1926991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 1927991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * FIXME Remove validation from scope of lock. 1928991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 1929991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mutex_lock(&dm_thin_pool_table.mutex); 1930991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1931991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (argc < 4) { 1932991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ti->error = "Invalid argument count"; 1933991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = -EINVAL; 1934991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber goto out_unlock; 1935991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1936991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber as.argc = argc; 1937991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber as.argv = argv; 1938991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1939991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_get_device(ti, argv[0], FMODE_READ | FMODE_WRITE, &metadata_dev); 1940991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) { 1941991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ti->error = "Error opening metadata block device"; 1942991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber goto out_unlock; 1943991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1944991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1945991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber metadata_dev_size = i_size_read(metadata_dev->bdev->bd_inode) >> SECTOR_SHIFT; 1946c4a69ecdb463a901b4645230613961e134e897cdMike Snitzer if (metadata_dev_size > THIN_METADATA_MAX_SECTORS_WARNING) 1947c4a69ecdb463a901b4645230613961e134e897cdMike Snitzer DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.", 1948c4a69ecdb463a901b4645230613961e134e897cdMike Snitzer bdevname(metadata_dev->bdev, b), THIN_METADATA_MAX_SECTORS); 1949991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1950991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_get_device(ti, argv[1], FMODE_READ | FMODE_WRITE, &data_dev); 1951991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) { 1952991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ti->error = "Error getting data device"; 1953991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber goto out_metadata; 1954991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1955991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1956991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (kstrtoul(argv[2], 10, &block_size) || !block_size || 1957991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS || 1958991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS || 1959991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber !is_power_of_2(block_size)) { 1960991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ti->error = "Invalid block size"; 1961991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = -EINVAL; 1962991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber goto out; 1963991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1964991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1965991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (kstrtoull(argv[3], 10, (unsigned long long *)&low_water_blocks)) { 1966991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ti->error = "Invalid low water mark"; 1967991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = -EINVAL; 1968991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber goto out; 1969991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1970991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1971991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 1972991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Set default pool features. 1973991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 197467e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber pool_features_init(&pf); 1975991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1976991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_consume_args(&as, 4); 1977991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = parse_pool_features(&as, &pf, ti); 1978991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 1979991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber goto out; 1980991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1981991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pt = kzalloc(sizeof(*pt), GFP_KERNEL); 1982991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!pt) { 1983991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = -ENOMEM; 1984991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber goto out; 1985991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1986991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 1987991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool = __pool_find(dm_table_get_md(ti->table), metadata_dev->bdev, 198867e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber block_size, &ti->error, &pool_created); 1989991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (IS_ERR(pool)) { 1990991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = PTR_ERR(pool); 1991991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber goto out_free_pt; 1992991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 1993991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 199467e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber /* 199567e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber * 'pool_created' reflects whether this is the first table load. 199667e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber * Top level discard support is not allowed to be changed after 199767e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber * initial load. This would require a pool reload to trigger thin 199867e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber * device changes. 199967e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber */ 200067e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber if (!pool_created && pf.discard_enabled != pool->pf.discard_enabled) { 200167e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber ti->error = "Discard support cannot be disabled once enabled"; 200267e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber r = -EINVAL; 200367e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber goto out_flags_changed; 200467e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber } 200567e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber 2006991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pt->pool = pool; 2007991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pt->ti = ti; 2008991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pt->metadata_dev = metadata_dev; 2009991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pt->data_dev = data_dev; 2010991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pt->low_water_blocks = low_water_blocks; 201167e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber pt->pf = pf; 2012991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ti->num_flush_requests = 1; 201367e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber /* 201467e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber * Only need to enable discards if the pool should pass 201567e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber * them down to the data device. The thin device's discard 201667e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber * processing will cause mappings to be removed from the btree. 201767e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber */ 201867e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber if (pf.discard_enabled && pf.discard_passdown) { 201967e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber ti->num_discard_requests = 1; 202067e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber /* 202167e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber * Setting 'discards_supported' circumvents the normal 202267e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber * stacking of discard limits (this keeps the pool and 202367e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber * thin devices' discard limits consistent). 202467e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber */ 202567e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber ti->discards_supported = 1; 202667e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber } 2027991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ti->private = pt; 2028991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2029991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pt->callbacks.congested_fn = pool_is_congested; 2030991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_table_add_target_callbacks(ti->table, &pt->callbacks); 2031991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2032991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mutex_unlock(&dm_thin_pool_table.mutex); 2033991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2034991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return 0; 2035991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 203667e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornberout_flags_changed: 203767e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber __pool_dec(pool); 2038991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberout_free_pt: 2039991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber kfree(pt); 2040991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberout: 2041991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_put_device(ti, data_dev); 2042991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberout_metadata: 2043991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_put_device(ti, metadata_dev); 2044991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberout_unlock: 2045991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mutex_unlock(&dm_thin_pool_table.mutex); 2046991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2047991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2048991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2049991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2050991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int pool_map(struct dm_target *ti, struct bio *bio, 2051991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber union map_info *map_context) 2052991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2053991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 2054991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool_c *pt = ti->private; 2055991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = pt->pool; 2056991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long flags; 2057991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2058991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 2059991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * As this is a singleton target, ti->begin is always zero. 2060991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 2061991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&pool->lock, flags); 2062991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bio->bi_bdev = pt->data_dev->bdev; 2063991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = DM_MAPIO_REMAPPED; 2064991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&pool->lock, flags); 2065991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2066991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2067991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2068991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2069991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 2070991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Retrieves the number of blocks of the data device from 2071991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * the superblock and compares it to the actual device size, 2072991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * thus resizing the data device in case it has grown. 2073991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 2074991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * This both copes with opening preallocated data devices in the ctr 2075991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * being followed by a resume 2076991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * -and- 2077991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * calling the resume method individually after userspace has 2078991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * grown the data device in reaction to a table event. 2079991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 2080991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int pool_preresume(struct dm_target *ti) 2081991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2082991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 2083991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool_c *pt = ti->private; 2084991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = pt->pool; 2085991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t data_size, sb_data_size; 2086991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2087991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 2088991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Take control of the pool object. 2089991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 2090991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = bind_control_target(pool, ti); 2091991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 2092991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2093991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2094991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber data_size = ti->len >> pool->block_shift; 2095991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_pool_get_data_dev_size(pool->pmd, &sb_data_size); 2096991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) { 2097991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMERR("failed to retrieve data device size"); 2098991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2099991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2100991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2101991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (data_size < sb_data_size) { 2102991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMERR("pool target too small, is %llu blocks (expected %llu)", 2103991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber data_size, sb_data_size); 2104991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return -EINVAL; 2105991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2106991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } else if (data_size > sb_data_size) { 2107991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_pool_resize_data_dev(pool->pmd, data_size); 2108991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) { 2109991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMERR("failed to resize data device"); 2110991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2111991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2112991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2113991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_pool_commit_metadata(pool->pmd); 2114991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) { 2115991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMERR("%s: dm_pool_commit_metadata() failed, error = %d", 2116991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber __func__, r); 2117991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2118991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2119991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2120991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2121991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return 0; 2122991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2123991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2124991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void pool_resume(struct dm_target *ti) 2125991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2126991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool_c *pt = ti->private; 2127991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = pt->pool; 2128991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned long flags; 2129991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2130991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_lock_irqsave(&pool->lock, flags); 2131991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->low_water_triggered = 0; 2132991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool->no_free_space = 0; 2133991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber __requeue_bios(pool); 2134991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber spin_unlock_irqrestore(&pool->lock, flags); 2135991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2136905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber do_waker(&pool->waker.work); 2137991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2138991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2139991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void pool_postsuspend(struct dm_target *ti) 2140991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2141991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 2142991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool_c *pt = ti->private; 2143991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = pt->pool; 2144991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2145905e51b39a5558706a6ed883fe104de3d417050bJoe Thornber cancel_delayed_work(&pool->waker); 2146991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber flush_workqueue(pool->wq); 2147991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2148991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_pool_commit_metadata(pool->pmd); 2149991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r < 0) { 2150991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMERR("%s: dm_pool_commit_metadata() failed, error = %d", 2151991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber __func__, r); 2152991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* FIXME: invalidate device? error the next FUA or FLUSH bio ?*/ 2153991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2154991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2155991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2156991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int check_arg_count(unsigned argc, unsigned args_required) 2157991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2158991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (argc != args_required) { 2159991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMWARN("Message received with %u arguments instead of %u.", 2160991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber argc, args_required); 2161991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return -EINVAL; 2162991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2163991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2164991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return 0; 2165991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2166991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2167991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int read_dev_id(char *arg, dm_thin_id *dev_id, int warning) 2168991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2169991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!kstrtoull(arg, 10, (unsigned long long *)dev_id) && 2170991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber *dev_id <= MAX_DEV_ID) 2171991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return 0; 2172991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2173991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (warning) 2174991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMWARN("Message received with invalid device id: %s", arg); 2175991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2176991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return -EINVAL; 2177991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2178991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2179991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int process_create_thin_mesg(unsigned argc, char **argv, struct pool *pool) 2180991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2181991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_thin_id dev_id; 2182991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 2183991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2184991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = check_arg_count(argc, 2); 2185991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 2186991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2187991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2188991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = read_dev_id(argv[1], &dev_id, 1); 2189991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 2190991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2191991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2192991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_pool_create_thin(pool->pmd, dev_id); 2193991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) { 2194991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMWARN("Creation of new thinly-provisioned device with id %s failed.", 2195991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber argv[1]); 2196991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2197991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2198991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2199991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return 0; 2200991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2201991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2202991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int process_create_snap_mesg(unsigned argc, char **argv, struct pool *pool) 2203991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2204991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_thin_id dev_id; 2205991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_thin_id origin_dev_id; 2206991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 2207991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2208991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = check_arg_count(argc, 3); 2209991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 2210991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2211991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2212991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = read_dev_id(argv[1], &dev_id, 1); 2213991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 2214991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2215991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2216991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = read_dev_id(argv[2], &origin_dev_id, 1); 2217991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 2218991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2219991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2220991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_pool_create_snap(pool->pmd, dev_id, origin_dev_id); 2221991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) { 2222991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMWARN("Creation of new snapshot %s of device %s failed.", 2223991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber argv[1], argv[2]); 2224991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2225991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2226991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2227991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return 0; 2228991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2229991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2230991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int process_delete_mesg(unsigned argc, char **argv, struct pool *pool) 2231991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2232991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_thin_id dev_id; 2233991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 2234991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2235991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = check_arg_count(argc, 2); 2236991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 2237991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2238991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2239991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = read_dev_id(argv[1], &dev_id, 1); 2240991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 2241991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2242991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2243991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_pool_delete_thin_device(pool->pmd, dev_id); 2244991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 2245991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMWARN("Deletion of thin device %s failed.", argv[1]); 2246991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2247991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2248991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2249991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2250991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int process_set_transaction_id_mesg(unsigned argc, char **argv, struct pool *pool) 2251991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2252991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_thin_id old_id, new_id; 2253991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 2254991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2255991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = check_arg_count(argc, 3); 2256991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 2257991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2258991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2259991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (kstrtoull(argv[1], 10, (unsigned long long *)&old_id)) { 2260991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMWARN("set_transaction_id message: Unrecognised id %s.", argv[1]); 2261991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return -EINVAL; 2262991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2263991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2264991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (kstrtoull(argv[2], 10, (unsigned long long *)&new_id)) { 2265991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMWARN("set_transaction_id message: Unrecognised new id %s.", argv[2]); 2266991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return -EINVAL; 2267991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2268991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2269991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_pool_set_metadata_transaction_id(pool->pmd, old_id, new_id); 2270991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) { 2271991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMWARN("Failed to change transaction id from %s to %s.", 2272991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber argv[1], argv[2]); 2273991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2274991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2275991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2276991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return 0; 2277991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2278991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2279991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 2280991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Messages supported: 2281991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * create_thin <dev_id> 2282991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * create_snap <dev_id> <origin_id> 2283991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * delete <dev_id> 2284991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * trim <dev_id> <new_size_in_sectors> 2285991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * set_transaction_id <current_trans_id> <new_trans_id> 2286991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 2287991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int pool_message(struct dm_target *ti, unsigned argc, char **argv) 2288991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2289991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r = -EINVAL; 2290991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool_c *pt = ti->private; 2291991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = pt->pool; 2292991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2293991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!strcasecmp(argv[0], "create_thin")) 2294991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = process_create_thin_mesg(argc, argv, pool); 2295991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2296991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber else if (!strcasecmp(argv[0], "create_snap")) 2297991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = process_create_snap_mesg(argc, argv, pool); 2298991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2299991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber else if (!strcasecmp(argv[0], "delete")) 2300991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = process_delete_mesg(argc, argv, pool); 2301991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2302991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber else if (!strcasecmp(argv[0], "set_transaction_id")) 2303991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = process_set_transaction_id_mesg(argc, argv, pool); 2304991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2305991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber else 2306991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMWARN("Unrecognised thin pool target message received: %s", argv[0]); 2307991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2308991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!r) { 2309991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_pool_commit_metadata(pool->pmd); 2310991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 2311991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMERR("%s message: dm_pool_commit_metadata() failed, error = %d", 2312991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber argv[0], r); 2313991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2314991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2315991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2316991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2317991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2318991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 2319991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Status line is: 2320991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * <transaction id> <used metadata sectors>/<total metadata sectors> 2321991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * <used data sectors>/<total data sectors> <held metadata root> 2322991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 2323991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int pool_status(struct dm_target *ti, status_type_t type, 2324991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber char *result, unsigned maxlen) 2325991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 232667e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber int r, count; 2327991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber unsigned sz = 0; 2328991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber uint64_t transaction_id; 2329991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t nr_free_blocks_data; 2330991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t nr_free_blocks_metadata; 2331991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t nr_blocks_data; 2332991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t nr_blocks_metadata; 2333991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t held_root; 2334991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber char buf[BDEVNAME_SIZE]; 2335991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber char buf2[BDEVNAME_SIZE]; 2336991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool_c *pt = ti->private; 2337991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = pt->pool; 2338991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2339991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber switch (type) { 2340991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber case STATUSTYPE_INFO: 2341991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_pool_get_metadata_transaction_id(pool->pmd, 2342991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber &transaction_id); 2343991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 2344991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2345991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2346991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_pool_get_free_metadata_block_count(pool->pmd, 2347991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber &nr_free_blocks_metadata); 2348991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 2349991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2350991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2351991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_pool_get_metadata_dev_size(pool->pmd, &nr_blocks_metadata); 2352991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 2353991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2354991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2355991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_pool_get_free_block_count(pool->pmd, 2356991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber &nr_free_blocks_data); 2357991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 2358991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2359991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2360991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_pool_get_data_dev_size(pool->pmd, &nr_blocks_data); 2361991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 2362991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2363991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2364991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_pool_get_held_metadata_root(pool->pmd, &held_root); 2365991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 2366991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2367991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2368991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMEMIT("%llu %llu/%llu %llu/%llu ", 2369991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber (unsigned long long)transaction_id, 2370991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata), 2371991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber (unsigned long long)nr_blocks_metadata, 2372991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber (unsigned long long)(nr_blocks_data - nr_free_blocks_data), 2373991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber (unsigned long long)nr_blocks_data); 2374991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2375991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (held_root) 2376991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMEMIT("%llu", held_root); 2377991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber else 2378991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMEMIT("-"); 2379991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2380991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber break; 2381991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2382991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber case STATUSTYPE_TABLE: 2383991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMEMIT("%s %s %lu %llu ", 2384991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber format_dev_t(buf, pt->metadata_dev->bdev->bd_dev), 2385991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber format_dev_t(buf2, pt->data_dev->bdev->bd_dev), 2386991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber (unsigned long)pool->sectors_per_block, 2387991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber (unsigned long long)pt->low_water_blocks); 2388991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 238967e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber count = !pool->pf.zero_new_blocks + !pool->pf.discard_enabled + 2390f402693d06f32e746c6153e459c5fb064fa02741Mike Snitzer !pt->pf.discard_passdown; 239167e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber DMEMIT("%u ", count); 2392991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 239367e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber if (!pool->pf.zero_new_blocks) 2394991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMEMIT("skip_block_zeroing "); 239567e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber 239667e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber if (!pool->pf.discard_enabled) 239767e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber DMEMIT("ignore_discard "); 239867e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber 2399f402693d06f32e746c6153e459c5fb064fa02741Mike Snitzer if (!pt->pf.discard_passdown) 240067e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber DMEMIT("no_discard_passdown "); 240167e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber 2402991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber break; 2403991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2404991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2405991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return 0; 2406991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2407991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2408991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int pool_iterate_devices(struct dm_target *ti, 2409991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber iterate_devices_callout_fn fn, void *data) 2410991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2411991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool_c *pt = ti->private; 2412991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2413991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return fn(ti, pt->data_dev, 0, ti->len, data); 2414991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2415991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2416991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int pool_merge(struct dm_target *ti, struct bvec_merge_data *bvm, 2417991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct bio_vec *biovec, int max_size) 2418991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2419991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool_c *pt = ti->private; 2420991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct request_queue *q = bdev_get_queue(pt->data_dev->bdev); 2421991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2422991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!q->merge_bvec_fn) 2423991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return max_size; 2424991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2425991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber bvm->bi_bdev = pt->data_dev->bdev; 2426991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2427991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); 2428991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2429991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2430104655fd4dcebd50068ef30253a001da72e3a081Joe Thornberstatic void set_discard_limits(struct pool *pool, struct queue_limits *limits) 2431104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber{ 243267e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber /* 243367e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber * FIXME: these limits may be incompatible with the pool's data device 243467e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber */ 2435104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber limits->max_discard_sectors = pool->sectors_per_block; 2436104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber 2437104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber /* 2438104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber * This is just a hint, and not enforced. We have to cope with 2439104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber * bios that overlap 2 blocks. 2440104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber */ 2441104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber limits->discard_granularity = pool->sectors_per_block << SECTOR_SHIFT; 244267e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber limits->discard_zeroes_data = pool->pf.zero_new_blocks; 2443104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber} 2444104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber 2445991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void pool_io_hints(struct dm_target *ti, struct queue_limits *limits) 2446991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2447991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool_c *pt = ti->private; 2448991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct pool *pool = pt->pool; 2449991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2450991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber blk_limits_io_min(limits, 0); 2451991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber blk_limits_io_opt(limits, pool->sectors_per_block << SECTOR_SHIFT); 245267e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber if (pool->pf.discard_enabled) 245367e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber set_discard_limits(pool, limits); 2454991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2455991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2456991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic struct target_type pool_target = { 2457991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .name = "thin-pool", 2458991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .features = DM_TARGET_SINGLETON | DM_TARGET_ALWAYS_WRITEABLE | 2459991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DM_TARGET_IMMUTABLE, 246067e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber .version = {1, 1, 0}, 2461991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .module = THIS_MODULE, 2462991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .ctr = pool_ctr, 2463991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .dtr = pool_dtr, 2464991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .map = pool_map, 2465991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .postsuspend = pool_postsuspend, 2466991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .preresume = pool_preresume, 2467991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .resume = pool_resume, 2468991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .message = pool_message, 2469991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .status = pool_status, 2470991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .merge = pool_merge, 2471991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .iterate_devices = pool_iterate_devices, 2472991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .io_hints = pool_io_hints, 2473991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber}; 2474991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2475991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/*---------------------------------------------------------------- 2476991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Thin target methods 2477991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber *--------------------------------------------------------------*/ 2478991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void thin_dtr(struct dm_target *ti) 2479991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2480991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct thin_c *tc = ti->private; 2481991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2482991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mutex_lock(&dm_thin_pool_table.mutex); 2483991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2484991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber __pool_dec(tc->pool); 2485991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_pool_close_thin_device(tc->td); 2486991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_put_device(ti, tc->pool_dev); 24872dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber if (tc->origin_dev) 24882dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber dm_put_device(ti, tc->origin_dev); 2489991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber kfree(tc); 2490991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2491991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mutex_unlock(&dm_thin_pool_table.mutex); 2492991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2493991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2494991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 2495991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * Thin target parameters: 2496991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 24972dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber * <pool_dev> <dev_id> [origin_dev] 2498991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * 2499991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * pool_dev: the path to the pool (eg, /dev/mapper/my_pool) 2500991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * dev_id: the internal device identifier 25012dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber * origin_dev: a device external to the pool that should act as the origin 250267e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber * 250367e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber * If the pool device has discards disabled, they get disabled for the thin 250467e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber * device as well. 2505991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 2506991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int thin_ctr(struct dm_target *ti, unsigned argc, char **argv) 2507991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2508991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 2509991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct thin_c *tc; 25102dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber struct dm_dev *pool_dev, *origin_dev; 2511991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct mapped_device *pool_md; 2512991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2513991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mutex_lock(&dm_thin_pool_table.mutex); 2514991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 25152dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber if (argc != 2 && argc != 3) { 2516991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ti->error = "Invalid argument count"; 2517991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = -EINVAL; 2518991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber goto out_unlock; 2519991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2520991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2521991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber tc = ti->private = kzalloc(sizeof(*tc), GFP_KERNEL); 2522991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!tc) { 2523991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ti->error = "Out of memory"; 2524991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = -ENOMEM; 2525991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber goto out_unlock; 2526991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2527991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 25282dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber if (argc == 3) { 25292dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber r = dm_get_device(ti, argv[2], FMODE_READ, &origin_dev); 25302dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber if (r) { 25312dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber ti->error = "Error opening origin device"; 25322dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber goto bad_origin_dev; 25332dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber } 25342dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber tc->origin_dev = origin_dev; 25352dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber } 25362dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber 2537991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &pool_dev); 2538991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) { 2539991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ti->error = "Error opening pool device"; 2540991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber goto bad_pool_dev; 2541991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2542991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber tc->pool_dev = pool_dev; 2543991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2544991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (read_dev_id(argv[1], (unsigned long long *)&tc->dev_id, 0)) { 2545991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ti->error = "Invalid device id"; 2546991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = -EINVAL; 2547991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber goto bad_common; 2548991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2549991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2550991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool_md = dm_get_md(tc->pool_dev->bdev->bd_dev); 2551991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!pool_md) { 2552991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ti->error = "Couldn't get pool mapped device"; 2553991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = -EINVAL; 2554991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber goto bad_common; 2555991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2556991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2557991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber tc->pool = __pool_table_lookup(pool_md); 2558991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!tc->pool) { 2559991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ti->error = "Couldn't find pool object"; 2560991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = -EINVAL; 2561991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber goto bad_pool_lookup; 2562991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2563991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber __pool_inc(tc->pool); 2564991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2565991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_pool_open_thin_device(tc->pool->pmd, tc->dev_id, &tc->td); 2566991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) { 2567991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ti->error = "Couldn't open thin internal device"; 2568991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber goto bad_thin_open; 2569991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2570991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2571991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ti->split_io = tc->pool->sectors_per_block; 2572991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ti->num_flush_requests = 1; 257367e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber 257467e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber /* In case the pool supports discards, pass them on. */ 257567e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber if (tc->pool->pf.discard_enabled) { 257667e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber ti->discards_supported = 1; 257767e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber ti->num_discard_requests = 1; 257867e2e2b281812b5caf4923a38aadc6b89e34f064Joe Thornber } 2579991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2580991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_put(pool_md); 2581991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2582991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mutex_unlock(&dm_thin_pool_table.mutex); 2583991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2584991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return 0; 2585991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2586991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberbad_thin_open: 2587991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber __pool_dec(tc->pool); 2588991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberbad_pool_lookup: 2589991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_put(pool_md); 2590991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberbad_common: 2591991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_put_device(ti, tc->pool_dev); 2592991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberbad_pool_dev: 25932dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber if (tc->origin_dev) 25942dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber dm_put_device(ti, tc->origin_dev); 25952dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornberbad_origin_dev: 2596991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber kfree(tc); 2597991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberout_unlock: 2598991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber mutex_unlock(&dm_thin_pool_table.mutex); 2599991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2600991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2601991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2602991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2603991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int thin_map(struct dm_target *ti, struct bio *bio, 2604991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber union map_info *map_context) 2605991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 26066efd6e83092cd4a7532270bc843de90bb93f6683Alasdair G Kergon bio->bi_sector = dm_target_offset(ti, bio->bi_sector); 2607991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2608991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return thin_bio_map(ti, bio, map_context); 2609991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2610991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2611eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornberstatic int thin_endio(struct dm_target *ti, 2612eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber struct bio *bio, int err, 2613eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber union map_info *map_context) 2614eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber{ 2615eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber unsigned long flags; 2616eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber struct endio_hook *h = map_context->ptr; 2617eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber struct list_head work; 2618eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber struct new_mapping *m, *tmp; 2619eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber struct pool *pool = h->tc->pool; 2620eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber 2621eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber if (h->shared_read_entry) { 2622eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber INIT_LIST_HEAD(&work); 2623eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber ds_dec(h->shared_read_entry, &work); 2624eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber 2625eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber spin_lock_irqsave(&pool->lock, flags); 2626eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber list_for_each_entry_safe(m, tmp, &work, list) { 2627eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber list_del(&m->list); 2628eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber m->quiesced = 1; 2629eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber __maybe_add_mapping(m); 2630eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber } 2631eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber spin_unlock_irqrestore(&pool->lock, flags); 2632eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber } 2633eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber 2634104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber if (h->all_io_entry) { 2635104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber INIT_LIST_HEAD(&work); 2636104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber ds_dec(h->all_io_entry, &work); 2637c3a0ce2eab76daf9516c817c3f227ea3f4549bd8Mike Snitzer spin_lock_irqsave(&pool->lock, flags); 2638104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber list_for_each_entry_safe(m, tmp, &work, list) 2639104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber list_add(&m->list, &pool->prepared_discards); 2640c3a0ce2eab76daf9516c817c3f227ea3f4549bd8Mike Snitzer spin_unlock_irqrestore(&pool->lock, flags); 2641104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber } 2642104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber 2643eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber mempool_free(h, pool->endio_hook_pool); 2644eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber 2645eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber return 0; 2646eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber} 2647eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber 2648991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void thin_postsuspend(struct dm_target *ti) 2649991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2650991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (dm_noflush_suspending(ti)) 2651991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber requeue_io((struct thin_c *)ti->private); 2652991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2653991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2654991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/* 2655991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * <nr mapped sectors> <highest mapped sector> 2656991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 2657991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int thin_status(struct dm_target *ti, status_type_t type, 2658991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber char *result, unsigned maxlen) 2659991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2660991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 2661991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber ssize_t sz = 0; 2662991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t mapped, highest; 2663991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber char buf[BDEVNAME_SIZE]; 2664991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct thin_c *tc = ti->private; 2665991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2666991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!tc->td) 2667991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMEMIT("-"); 2668991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber else { 2669991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber switch (type) { 2670991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber case STATUSTYPE_INFO: 2671991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_thin_get_mapped_count(tc->td, &mapped); 2672991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 2673991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2674991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2675991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_thin_get_highest_mapped_block(tc->td, &highest); 2676991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r < 0) 2677991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2678991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2679991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMEMIT("%llu ", mapped * tc->pool->sectors_per_block); 2680991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 2681991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMEMIT("%llu", ((highest + 1) * 2682991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber tc->pool->sectors_per_block) - 1); 2683991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber else 2684991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMEMIT("-"); 2685991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber break; 2686991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2687991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber case STATUSTYPE_TABLE: 2688991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber DMEMIT("%s %lu", 2689991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber format_dev_t(buf, tc->pool_dev->bdev->bd_dev), 2690991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber (unsigned long) tc->dev_id); 26912dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber if (tc->origin_dev) 26922dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber DMEMIT(" %s", format_dev_t(buf, tc->origin_dev->bdev->bd_dev)); 2693991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber break; 2694991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2695991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber } 2696991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2697991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return 0; 2698991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2699991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2700991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int thin_iterate_devices(struct dm_target *ti, 2701991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber iterate_devices_callout_fn fn, void *data) 2702991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2703991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_block_t blocks; 2704991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct thin_c *tc = ti->private; 2705991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2706991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber /* 2707991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * We can't call dm_pool_get_data_dev_size() since that blocks. So 2708991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber * we follow a more convoluted path through to the pool's target. 2709991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber */ 2710991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (!tc->pool->ti) 2711991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return 0; /* nothing is bound */ 2712991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2713991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber blocks = tc->pool->ti->len >> tc->pool->block_shift; 2714991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (blocks) 2715991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return fn(ti, tc->pool_dev, 0, tc->pool->sectors_per_block * blocks, data); 2716991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2717991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return 0; 2718991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2719991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2720991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void thin_io_hints(struct dm_target *ti, struct queue_limits *limits) 2721991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2722991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber struct thin_c *tc = ti->private; 2723104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber struct pool *pool = tc->pool; 2724991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2725991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber blk_limits_io_min(limits, 0); 2726104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber blk_limits_io_opt(limits, pool->sectors_per_block << SECTOR_SHIFT); 2727104655fd4dcebd50068ef30253a001da72e3a081Joe Thornber set_discard_limits(pool, limits); 2728991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2729991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2730991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic struct target_type thin_target = { 2731991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .name = "thin", 27322dd9c257fbc243aa76ee6db0bb8371f9f74fad2dJoe Thornber .version = {1, 1, 0}, 2733991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .module = THIS_MODULE, 2734991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .ctr = thin_ctr, 2735991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .dtr = thin_dtr, 2736991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .map = thin_map, 2737eb2aa48d4eb7aee63cba201bf47641dad3e92250Joe Thornber .end_io = thin_endio, 2738991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .postsuspend = thin_postsuspend, 2739991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .status = thin_status, 2740991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .iterate_devices = thin_iterate_devices, 2741991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber .io_hints = thin_io_hints, 2742991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber}; 2743991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2744991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber/*----------------------------------------------------------------*/ 2745991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2746991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic int __init dm_thin_init(void) 2747991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2748991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber int r; 2749991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2750991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber pool_table_init(); 2751991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2752991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_register_target(&thin_target); 2753991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 2754991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2755991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2756991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber r = dm_register_target(&pool_target); 2757991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber if (r) 2758991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_unregister_target(&thin_target); 2759991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2760991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber return r; 2761991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2762991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2763991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornberstatic void dm_thin_exit(void) 2764991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber{ 2765991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_unregister_target(&thin_target); 2766991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber dm_unregister_target(&pool_target); 2767991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber} 2768991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 2769991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornbermodule_init(dm_thin_init); 2770991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornbermodule_exit(dm_thin_exit); 2771991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe Thornber 27727cab8bf1601d38fd73a0a4ea40cc4892b26907ccAlasdair G KergonMODULE_DESCRIPTION(DM_NAME " thin provisioning target"); 2773991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe ThornberMODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>"); 2774991d9fa02da0dd1f843dc011376965e0c8c6c9b5Joe ThornberMODULE_LICENSE("GPL"); 2775