dm-region-hash.c revision a72986c562eeec3f7b992198c168f0f41606fe53
11f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen/* 21f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Copyright (C) 2003 Sistina Software Limited. 31f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. 41f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 51f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * This file is released under the GPL. 61f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 71f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 81f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#include <linux/dm-dirty-log.h> 91f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#include <linux/dm-region-hash.h> 101f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 111f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#include <linux/ctype.h> 121f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#include <linux/init.h> 131f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#include <linux/module.h> 141f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#include <linux/vmalloc.h> 151f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 161f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#include "dm.h" 171f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 181f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#define DM_MSG_PREFIX "region hash" 191f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 201f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen/*----------------------------------------------------------------- 211f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Region hash 221f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 231f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * The mirror splits itself up into discrete regions. Each 241f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * region can be in one of three states: clean, dirty, 251f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * nosync. There is no need to put clean regions in the hash. 261f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 271f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * In addition to being present in the hash table a region _may_ 281f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * be present on one of three lists. 291f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 301f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * clean_regions: Regions on this list have no io pending to 311f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * them, they are in sync, we are no longer interested in them, 321f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * they are dull. dm_rh_update_states() will remove them from the 331f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * hash table. 341f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 351f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * quiesced_regions: These regions have been spun down, ready 361f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * for recovery. rh_recovery_start() will remove regions from 371f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * this list and hand them to kmirrord, which will schedule the 381f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * recovery io with kcopyd. 391f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 401f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * recovered_regions: Regions that kcopyd has successfully 411f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * recovered. dm_rh_update_states() will now schedule any delayed 421f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * io, up the recovery_count, and remove the region from the 431f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * hash. 441f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 451f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * There are 2 locks: 461f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * A rw spin lock 'hash_lock' protects just the hash table, 471f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * this is never held in write mode from interrupt context, 481f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * which I believe means that we only have to disable irqs when 491f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * doing a write lock. 501f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 511f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * An ordinary spin lock 'region_lock' that protects the three 521f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * lists in the region_hash, with the 'state', 'list' and 531f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 'delayed_bios' fields of the regions. This is used from irq 541f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * context, so all other uses will have to suspend local irqs. 551f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen *---------------------------------------------------------------*/ 561f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstruct dm_region_hash { 571f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen uint32_t region_size; 581f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen unsigned region_shift; 591f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 601f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* holds persistent region state */ 611f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_dirty_log *log; 621f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 631f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* hash table */ 641f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rwlock_t hash_lock; 651f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen mempool_t *region_pool; 661f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen unsigned mask; 671f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen unsigned nr_buckets; 681f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen unsigned prime; 691f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen unsigned shift; 701f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct list_head *buckets; 711f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 721f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen unsigned max_recovery; /* Max # of regions to recover in parallel */ 731f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 741f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spinlock_t region_lock; 751f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen atomic_t recovery_in_flight; 761f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct semaphore recovery_count; 771f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct list_head clean_regions; 781f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct list_head quiesced_regions; 791f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct list_head recovered_regions; 801f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct list_head failed_recovered_regions; 811f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 821f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen void *context; 831f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen sector_t target_begin; 841f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 851f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* Callback function to schedule bios writes */ 861f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen void (*dispatch_bios)(void *context, struct bio_list *bios); 871f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 881f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* Callback function to wakeup callers worker thread. */ 891f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen void (*wakeup_workers)(void *context); 901f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 911f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* Callback function to wakeup callers recovery waiters. */ 921f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen void (*wakeup_all_recovery_waiters)(void *context); 931f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen}; 941f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 951f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstruct dm_region { 961f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region_hash *rh; /* FIXME: can we get rid of this ? */ 971f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen region_t key; 981f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen int state; 991f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1001f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct list_head hash_list; 1011f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct list_head list; 1021f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1031f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen atomic_t pending; 1041f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct bio_list delayed_bios; 1051f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen}; 1061f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1071f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen/* 1081f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Conversion fns 1091f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 1101f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstatic region_t dm_rh_sector_to_region(struct dm_region_hash *rh, sector_t sector) 1111f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 1121f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return sector >> rh->region_shift; 1131f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 1141f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1151f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagensector_t dm_rh_region_to_sector(struct dm_region_hash *rh, region_t region) 1161f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 1171f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return region << rh->region_shift; 1181f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 1191f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_region_to_sector); 1201f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1211f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenregion_t dm_rh_bio_to_region(struct dm_region_hash *rh, struct bio *bio) 1221f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 1231f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return dm_rh_sector_to_region(rh, bio->bi_sector - rh->target_begin); 1241f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 1251f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_bio_to_region); 1261f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1271f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenvoid *dm_rh_region_context(struct dm_region *reg) 1281f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 1291f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return reg->rh->context; 1301f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 1311f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_region_context); 1321f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1331f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenregion_t dm_rh_get_region_key(struct dm_region *reg) 1341f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 1351f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return reg->key; 1361f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 1371f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_get_region_key); 1381f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1391f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagensector_t dm_rh_get_region_size(struct dm_region_hash *rh) 1401f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 1411f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return rh->region_size; 1421f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 1431f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_get_region_size); 1441f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1451f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen/* 1461f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * FIXME: shall we pass in a structure instead of all these args to 1471f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * dm_region_hash_create()???? 1481f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 1491f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#define RH_HASH_MULT 2654435387U 1501f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#define RH_HASH_SHIFT 12 1511f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1521f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#define MIN_REGIONS 64 1531f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstruct dm_region_hash *dm_region_hash_create( 1541f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen void *context, void (*dispatch_bios)(void *context, 1551f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct bio_list *bios), 1561f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen void (*wakeup_workers)(void *context), 1571f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen void (*wakeup_all_recovery_waiters)(void *context), 1581f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen sector_t target_begin, unsigned max_recovery, 1591f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_dirty_log *log, uint32_t region_size, 1601f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen region_t nr_regions) 1611f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 1621f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region_hash *rh; 1631f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen unsigned nr_buckets, max_buckets; 1641f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen size_t i; 1651f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1661f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* 1671f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Calculate a suitable number of buckets for our hash 1681f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * table. 1691f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 1701f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen max_buckets = nr_regions >> 6; 1711f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen for (nr_buckets = 128u; nr_buckets < max_buckets; nr_buckets <<= 1) 1721f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen ; 1731f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen nr_buckets >>= 1; 1741f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1751f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh = kmalloc(sizeof(*rh), GFP_KERNEL); 1761f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (!rh) { 1771f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen DMERR("unable to allocate region hash memory"); 1781f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return ERR_PTR(-ENOMEM); 1791f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 1801f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1811f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->context = context; 1821f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->dispatch_bios = dispatch_bios; 1831f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->wakeup_workers = wakeup_workers; 1841f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->wakeup_all_recovery_waiters = wakeup_all_recovery_waiters; 1851f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->target_begin = target_begin; 1861f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->max_recovery = max_recovery; 1871f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->log = log; 1881f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->region_size = region_size; 1891f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->region_shift = ffs(region_size) - 1; 1901f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rwlock_init(&rh->hash_lock); 1911f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->mask = nr_buckets - 1; 1921f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->nr_buckets = nr_buckets; 1931f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1941f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->shift = RH_HASH_SHIFT; 1951f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->prime = RH_HASH_MULT; 1961f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1971f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->buckets = vmalloc(nr_buckets * sizeof(*rh->buckets)); 1981f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (!rh->buckets) { 1991f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen DMERR("unable to allocate region hash bucket memory"); 2001f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen kfree(rh); 2011f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return ERR_PTR(-ENOMEM); 2021f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 2031f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2041f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen for (i = 0; i < nr_buckets; i++) 2051f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen INIT_LIST_HEAD(rh->buckets + i); 2061f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2071f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_lock_init(&rh->region_lock); 2081f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen sema_init(&rh->recovery_count, 0); 2091f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen atomic_set(&rh->recovery_in_flight, 0); 2101f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen INIT_LIST_HEAD(&rh->clean_regions); 2111f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen INIT_LIST_HEAD(&rh->quiesced_regions); 2121f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen INIT_LIST_HEAD(&rh->recovered_regions); 2131f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen INIT_LIST_HEAD(&rh->failed_recovered_regions); 2141f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2151f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->region_pool = mempool_create_kmalloc_pool(MIN_REGIONS, 2161f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen sizeof(struct dm_region)); 2171f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (!rh->region_pool) { 2181f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen vfree(rh->buckets); 2191f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen kfree(rh); 2201f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh = ERR_PTR(-ENOMEM); 2211f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 2221f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2231f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return rh; 2241f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 2251f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_region_hash_create); 2261f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2271f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenvoid dm_region_hash_destroy(struct dm_region_hash *rh) 2281f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 2291f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen unsigned h; 2301f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg, *nreg; 2311f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2321f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen BUG_ON(!list_empty(&rh->quiesced_regions)); 2331f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen for (h = 0; h < rh->nr_buckets; h++) { 2341f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_for_each_entry_safe(reg, nreg, rh->buckets + h, 2351f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen hash_list) { 2361f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen BUG_ON(atomic_read(®->pending)); 2371f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen mempool_free(reg, rh->region_pool); 2381f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 2391f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 2401f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2411f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (rh->log) 2421f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen dm_dirty_log_destroy(rh->log); 2431f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2441f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (rh->region_pool) 2451f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen mempool_destroy(rh->region_pool); 2461f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2471f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen vfree(rh->buckets); 2481f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen kfree(rh); 2491f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 2501f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_region_hash_destroy); 2511f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2521f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstruct dm_dirty_log *dm_rh_dirty_log(struct dm_region_hash *rh) 2531f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 2541f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return rh->log; 2551f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 2561f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_dirty_log); 2571f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2581f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstatic unsigned rh_hash(struct dm_region_hash *rh, region_t region) 2591f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 2601f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return (unsigned) ((region * rh->prime) >> rh->shift) & rh->mask; 2611f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 2621f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2631f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstatic struct dm_region *__rh_lookup(struct dm_region_hash *rh, region_t region) 2641f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 2651f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg; 2661f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct list_head *bucket = rh->buckets + rh_hash(rh, region); 2671f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2681f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_for_each_entry(reg, bucket, hash_list) 2691f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (reg->key == region) 2701f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return reg; 2711f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2721f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return NULL; 2731f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 2741f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2751f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstatic void __rh_insert(struct dm_region_hash *rh, struct dm_region *reg) 2761f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 2771f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_add(®->hash_list, rh->buckets + rh_hash(rh, reg->key)); 2781f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 2791f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2801f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstatic struct dm_region *__rh_alloc(struct dm_region_hash *rh, region_t region) 2811f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 2821f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg, *nreg; 2831f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2841f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen nreg = mempool_alloc(rh->region_pool, GFP_ATOMIC); 2851f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (unlikely(!nreg)) 286a72986c562eeec3f7b992198c168f0f41606fe53Mikulas Patocka nreg = kmalloc(sizeof(*nreg), GFP_NOIO | __GFP_NOFAIL); 2871f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2881f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen nreg->state = rh->log->type->in_sync(rh->log, region, 1) ? 2891f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen DM_RH_CLEAN : DM_RH_NOSYNC; 2901f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen nreg->rh = rh; 2911f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen nreg->key = region; 2921f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen INIT_LIST_HEAD(&nreg->list); 2931f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen atomic_set(&nreg->pending, 0); 2941f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen bio_list_init(&nreg->delayed_bios); 2951f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2961f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen write_lock_irq(&rh->hash_lock); 2971f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = __rh_lookup(rh, region); 2981f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (reg) 2991f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* We lost the race. */ 3001f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen mempool_free(nreg, rh->region_pool); 3011f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen else { 3021f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen __rh_insert(rh, nreg); 3031f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (nreg->state == DM_RH_CLEAN) { 3041f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_lock(&rh->region_lock); 3051f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_add(&nreg->list, &rh->clean_regions); 3061f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_unlock(&rh->region_lock); 3071f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 3081f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3091f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = nreg; 3101f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 3111f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen write_unlock_irq(&rh->hash_lock); 3121f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3131f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return reg; 3141f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 3151f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3161f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstatic struct dm_region *__rh_find(struct dm_region_hash *rh, region_t region) 3171f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 3181f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg; 3191f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3201f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = __rh_lookup(rh, region); 3211f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (!reg) { 3221f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_unlock(&rh->hash_lock); 3231f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = __rh_alloc(rh, region); 3241f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_lock(&rh->hash_lock); 3251f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 3261f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3271f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return reg; 3281f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 3291f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3301f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenint dm_rh_get_state(struct dm_region_hash *rh, region_t region, int may_block) 3311f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 3321f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen int r; 3331f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg; 3341f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3351f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_lock(&rh->hash_lock); 3361f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = __rh_lookup(rh, region); 3371f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_unlock(&rh->hash_lock); 3381f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3391f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (reg) 3401f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return reg->state; 3411f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3421f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* 3431f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * The region wasn't in the hash, so we fall back to the 3441f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * dirty log. 3451f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 3461f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen r = rh->log->type->in_sync(rh->log, region, may_block); 3471f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3481f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* 3491f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Any error from the dirty log (eg. -EWOULDBLOCK) gets 3501f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * taken as a DM_RH_NOSYNC 3511f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 3521f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return r == 1 ? DM_RH_CLEAN : DM_RH_NOSYNC; 3531f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 3541f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_get_state); 3551f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3561f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstatic void complete_resync_work(struct dm_region *reg, int success) 3571f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 3581f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region_hash *rh = reg->rh; 3591f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3601f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->log->type->set_region_sync(rh->log, reg->key, success); 3611f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3621f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* 3631f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Dispatch the bios before we call 'wake_up_all'. 3641f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * This is important because if we are suspending, 3651f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * we want to know that recovery is complete and 3661f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * the work queue is flushed. If we wake_up_all 3671f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * before we dispatch_bios (queue bios and call wake()), 3681f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * then we risk suspending before the work queue 3691f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * has been properly flushed. 3701f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 3711f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->dispatch_bios(rh->context, ®->delayed_bios); 3721f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (atomic_dec_and_test(&rh->recovery_in_flight)) 3731f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->wakeup_all_recovery_waiters(rh->context); 3741f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen up(&rh->recovery_count); 3751f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 3761f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3771f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen/* dm_rh_mark_nosync 3781f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * @ms 3791f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * @bio 3801f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * @done 3811f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * @error 3821f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 3831f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * The bio was written on some mirror(s) but failed on other mirror(s). 3841f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * We can successfully endio the bio but should avoid the region being 3851f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * marked clean by setting the state DM_RH_NOSYNC. 3861f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 3871f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * This function is _not_ safe in interrupt context! 3881f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 3891f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenvoid dm_rh_mark_nosync(struct dm_region_hash *rh, 3901f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct bio *bio, unsigned done, int error) 3911f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 3921f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen unsigned long flags; 3931f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_dirty_log *log = rh->log; 3941f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg; 3951f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen region_t region = dm_rh_bio_to_region(rh, bio); 3961f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen int recovering = 0; 3971f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3981f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* We must inform the log that the sync count has changed. */ 3991f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen log->type->set_region_sync(log, region, 0); 4001f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4011f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_lock(&rh->hash_lock); 4021f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = __rh_find(rh, region); 4031f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_unlock(&rh->hash_lock); 4041f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4051f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* region hash entry should exist because write was in-flight */ 4061f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen BUG_ON(!reg); 4071f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen BUG_ON(!list_empty(®->list)); 4081f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4091f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_lock_irqsave(&rh->region_lock, flags); 4101f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* 4111f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Possible cases: 4121f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 1) DM_RH_DIRTY 4131f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 2) DM_RH_NOSYNC: was dirty, other preceeding writes failed 4141f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 3) DM_RH_RECOVERING: flushing pending writes 4151f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Either case, the region should have not been connected to list. 4161f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 4171f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen recovering = (reg->state == DM_RH_RECOVERING); 4181f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg->state = DM_RH_NOSYNC; 4191f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen BUG_ON(!list_empty(®->list)); 4201f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_unlock_irqrestore(&rh->region_lock, flags); 4211f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4221f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen bio_endio(bio, error); 4231f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (recovering) 4241f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen complete_resync_work(reg, 0); 4251f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 4261f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_mark_nosync); 4271f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4281f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenvoid dm_rh_update_states(struct dm_region_hash *rh, int errors_handled) 4291f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 4301f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg, *next; 4311f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4321f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen LIST_HEAD(clean); 4331f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen LIST_HEAD(recovered); 4341f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen LIST_HEAD(failed_recovered); 4351f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4361f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* 4371f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Quickly grab the lists. 4381f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 4391f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen write_lock_irq(&rh->hash_lock); 4401f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_lock(&rh->region_lock); 4411f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (!list_empty(&rh->clean_regions)) { 4421f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_splice_init(&rh->clean_regions, &clean); 4431f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4441f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_for_each_entry(reg, &clean, list) 4451f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_del(®->hash_list); 4461f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 4471f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4481f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (!list_empty(&rh->recovered_regions)) { 4491f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_splice_init(&rh->recovered_regions, &recovered); 4501f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4511f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_for_each_entry(reg, &recovered, list) 4521f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_del(®->hash_list); 4531f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 4541f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4551f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (!list_empty(&rh->failed_recovered_regions)) { 4561f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_splice_init(&rh->failed_recovered_regions, 4571f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen &failed_recovered); 4581f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4591f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_for_each_entry(reg, &failed_recovered, list) 4601f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_del(®->hash_list); 4611f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 4621f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4631f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_unlock(&rh->region_lock); 4641f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen write_unlock_irq(&rh->hash_lock); 4651f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4661f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* 4671f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * All the regions on the recovered and clean lists have 4681f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * now been pulled out of the system, so no need to do 4691f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * any more locking. 4701f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 4711f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_for_each_entry_safe(reg, next, &recovered, list) { 4721f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->log->type->clear_region(rh->log, reg->key); 4731f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen complete_resync_work(reg, 1); 4741f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen mempool_free(reg, rh->region_pool); 4751f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 4761f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4771f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_for_each_entry_safe(reg, next, &failed_recovered, list) { 4781f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen complete_resync_work(reg, errors_handled ? 0 : 1); 4791f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen mempool_free(reg, rh->region_pool); 4801f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 4811f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4821f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_for_each_entry_safe(reg, next, &clean, list) { 4831f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->log->type->clear_region(rh->log, reg->key); 4841f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen mempool_free(reg, rh->region_pool); 4851f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 4861f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4871f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->log->type->flush(rh->log); 4881f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 4891f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_update_states); 4901f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4911f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstatic void rh_inc(struct dm_region_hash *rh, region_t region) 4921f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 4931f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg; 4941f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4951f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_lock(&rh->hash_lock); 4961f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = __rh_find(rh, region); 4971f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4981f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_lock_irq(&rh->region_lock); 4991f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen atomic_inc(®->pending); 5001f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5011f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (reg->state == DM_RH_CLEAN) { 5021f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg->state = DM_RH_DIRTY; 5031f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_del_init(®->list); /* take off the clean list */ 5041f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_unlock_irq(&rh->region_lock); 5051f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5061f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->log->type->mark_region(rh->log, reg->key); 5071f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } else 5081f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_unlock_irq(&rh->region_lock); 5091f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5101f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5111f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_unlock(&rh->hash_lock); 5121f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 5131f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5141f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenvoid dm_rh_inc_pending(struct dm_region_hash *rh, struct bio_list *bios) 5151f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 5161f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct bio *bio; 5171f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5181f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen for (bio = bios->head; bio; bio = bio->bi_next) 5191f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh_inc(rh, dm_rh_bio_to_region(rh, bio)); 5201f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 5211f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_inc_pending); 5221f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5231f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenvoid dm_rh_dec(struct dm_region_hash *rh, region_t region) 5241f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 5251f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen unsigned long flags; 5261f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg; 5271f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen int should_wake = 0; 5281f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5291f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_lock(&rh->hash_lock); 5301f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = __rh_lookup(rh, region); 5311f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_unlock(&rh->hash_lock); 5321f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5331f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_lock_irqsave(&rh->region_lock, flags); 5341f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (atomic_dec_and_test(®->pending)) { 5351f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* 5361f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * There is no pending I/O for this region. 5371f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * We can move the region to corresponding list for next action. 5381f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * At this point, the region is not yet connected to any list. 5391f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 5401f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * If the state is DM_RH_NOSYNC, the region should be kept off 5411f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * from clean list. 5421f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * The hash entry for DM_RH_NOSYNC will remain in memory 5431f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * until the region is recovered or the map is reloaded. 5441f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 5451f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5461f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* do nothing for DM_RH_NOSYNC */ 5471f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (reg->state == DM_RH_RECOVERING) { 5481f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_add_tail(®->list, &rh->quiesced_regions); 5491f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } else if (reg->state == DM_RH_DIRTY) { 5501f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg->state = DM_RH_CLEAN; 5511f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_add(®->list, &rh->clean_regions); 5521f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 5531f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen should_wake = 1; 5541f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 5551f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_unlock_irqrestore(&rh->region_lock, flags); 5561f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5571f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (should_wake) 5581f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->wakeup_workers(rh->context); 5591f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 5601f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_dec); 5611f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5621f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen/* 5631f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Starts quiescing a region in preparation for recovery. 5641f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 5651f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstatic int __rh_recovery_prepare(struct dm_region_hash *rh) 5661f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 5671f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen int r; 5681f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen region_t region; 5691f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg; 5701f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5711f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* 5721f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Ask the dirty log what's next. 5731f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 5741f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen r = rh->log->type->get_resync_work(rh->log, ®ion); 5751f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (r <= 0) 5761f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return r; 5771f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5781f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* 5791f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Get this region, and start it quiescing by setting the 5801f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * recovering flag. 5811f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 5821f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_lock(&rh->hash_lock); 5831f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = __rh_find(rh, region); 5841f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_unlock(&rh->hash_lock); 5851f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5861f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_lock_irq(&rh->region_lock); 5871f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg->state = DM_RH_RECOVERING; 5881f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5891f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* Already quiesced ? */ 5901f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (atomic_read(®->pending)) 5911f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_del_init(®->list); 5921f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen else 5931f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_move(®->list, &rh->quiesced_regions); 5941f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5951f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_unlock_irq(&rh->region_lock); 5961f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5971f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return 1; 5981f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 5991f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6001f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenvoid dm_rh_recovery_prepare(struct dm_region_hash *rh) 6011f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 6021f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* Extra reference to avoid race with dm_rh_stop_recovery */ 6031f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen atomic_inc(&rh->recovery_in_flight); 6041f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6051f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen while (!down_trylock(&rh->recovery_count)) { 6061f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen atomic_inc(&rh->recovery_in_flight); 6071f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (__rh_recovery_prepare(rh) <= 0) { 6081f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen atomic_dec(&rh->recovery_in_flight); 6091f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen up(&rh->recovery_count); 6101f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen break; 6111f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 6121f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 6131f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6141f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* Drop the extra reference */ 6151f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (atomic_dec_and_test(&rh->recovery_in_flight)) 6161f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->wakeup_all_recovery_waiters(rh->context); 6171f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 6181f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_recovery_prepare); 6191f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6201f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen/* 6211f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Returns any quiesced regions. 6221f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 6231f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstruct dm_region *dm_rh_recovery_start(struct dm_region_hash *rh) 6241f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 6251f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg = NULL; 6261f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6271f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_lock_irq(&rh->region_lock); 6281f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (!list_empty(&rh->quiesced_regions)) { 6291f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = list_entry(rh->quiesced_regions.next, 6301f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region, list); 6311f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_del_init(®->list); /* remove from the quiesced list */ 6321f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 6331f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_unlock_irq(&rh->region_lock); 6341f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6351f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return reg; 6361f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 6371f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_recovery_start); 6381f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6391f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenvoid dm_rh_recovery_end(struct dm_region *reg, int success) 6401f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 6411f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region_hash *rh = reg->rh; 6421f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6431f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_lock_irq(&rh->region_lock); 6441f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (success) 6451f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_add(®->list, ®->rh->recovered_regions); 6461f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen else { 6471f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg->state = DM_RH_NOSYNC; 6481f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_add(®->list, ®->rh->failed_recovered_regions); 6491f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 6501f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_unlock_irq(&rh->region_lock); 6511f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6521f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->wakeup_workers(rh->context); 6531f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 6541f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_recovery_end); 6551f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6561f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen/* Return recovery in flight count. */ 6571f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenint dm_rh_recovery_in_flight(struct dm_region_hash *rh) 6581f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 6591f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return atomic_read(&rh->recovery_in_flight); 6601f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 6611f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_recovery_in_flight); 6621f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6631f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenint dm_rh_flush(struct dm_region_hash *rh) 6641f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 6651f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return rh->log->type->flush(rh->log); 6661f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 6671f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_flush); 6681f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6691f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenvoid dm_rh_delay(struct dm_region_hash *rh, struct bio *bio) 6701f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 6711f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg; 6721f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6731f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_lock(&rh->hash_lock); 6741f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = __rh_find(rh, dm_rh_bio_to_region(rh, bio)); 6751f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen bio_list_add(®->delayed_bios, bio); 6761f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_unlock(&rh->hash_lock); 6771f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 6781f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_delay); 6791f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6801f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenvoid dm_rh_stop_recovery(struct dm_region_hash *rh) 6811f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 6821f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen int i; 6831f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6841f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* wait for any recovering regions */ 6851f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen for (i = 0; i < rh->max_recovery; i++) 6861f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen down(&rh->recovery_count); 6871f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 6881f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_stop_recovery); 6891f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6901f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenvoid dm_rh_start_recovery(struct dm_region_hash *rh) 6911f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 6921f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen int i; 6931f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6941f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen for (i = 0; i < rh->max_recovery; i++) 6951f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen up(&rh->recovery_count); 6961f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6971f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->wakeup_workers(rh->context); 6981f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 6991f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_start_recovery); 7001f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 7011f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenMODULE_DESCRIPTION(DM_NAME " region hash"); 7021f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenMODULE_AUTHOR("Joe Thornber/Heinz Mauelshagen <dm-devel@redhat.com>"); 7031f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenMODULE_LICENSE("GPL"); 704