11f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen/* 21f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Copyright (C) 2003 Sistina Software Limited. 31f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. 41f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 51f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * This file is released under the GPL. 61f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 71f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 81f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#include <linux/dm-dirty-log.h> 91f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#include <linux/dm-region-hash.h> 101f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 111f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#include <linux/ctype.h> 121f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#include <linux/init.h> 131f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#include <linux/module.h> 145a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 151f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#include <linux/vmalloc.h> 161f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 171f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#include "dm.h" 181f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 191f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#define DM_MSG_PREFIX "region hash" 201f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 211f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen/*----------------------------------------------------------------- 221f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Region hash 231f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 241f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * The mirror splits itself up into discrete regions. Each 251f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * region can be in one of three states: clean, dirty, 261f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * nosync. There is no need to put clean regions in the hash. 271f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 281f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * In addition to being present in the hash table a region _may_ 291f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * be present on one of three lists. 301f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 311f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * clean_regions: Regions on this list have no io pending to 321f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * them, they are in sync, we are no longer interested in them, 331f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * they are dull. dm_rh_update_states() will remove them from the 341f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * hash table. 351f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 361f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * quiesced_regions: These regions have been spun down, ready 371f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * for recovery. rh_recovery_start() will remove regions from 381f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * this list and hand them to kmirrord, which will schedule the 391f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * recovery io with kcopyd. 401f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 411f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * recovered_regions: Regions that kcopyd has successfully 421f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * recovered. dm_rh_update_states() will now schedule any delayed 431f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * io, up the recovery_count, and remove the region from the 441f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * hash. 451f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 461f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * There are 2 locks: 471f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * A rw spin lock 'hash_lock' protects just the hash table, 481f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * this is never held in write mode from interrupt context, 491f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * which I believe means that we only have to disable irqs when 501f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * doing a write lock. 511f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 521f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * An ordinary spin lock 'region_lock' that protects the three 531f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * lists in the region_hash, with the 'state', 'list' and 541f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 'delayed_bios' fields of the regions. This is used from irq 551f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * context, so all other uses will have to suspend local irqs. 561f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen *---------------------------------------------------------------*/ 571f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstruct dm_region_hash { 581f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen uint32_t region_size; 591f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen unsigned region_shift; 601f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 611f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* holds persistent region state */ 621f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_dirty_log *log; 631f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 641f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* hash table */ 651f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rwlock_t hash_lock; 661f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen mempool_t *region_pool; 671f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen unsigned mask; 681f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen unsigned nr_buckets; 691f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen unsigned prime; 701f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen unsigned shift; 711f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct list_head *buckets; 721f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 731f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen unsigned max_recovery; /* Max # of regions to recover in parallel */ 741f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 751f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spinlock_t region_lock; 761f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen atomic_t recovery_in_flight; 771f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct semaphore recovery_count; 781f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct list_head clean_regions; 791f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct list_head quiesced_regions; 801f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct list_head recovered_regions; 811f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct list_head failed_recovered_regions; 821f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 834184153f9e483f9bb63339ed316e059962fe9794Mikulas Patocka /* 84d87f4c14f27dc82d215108d8392a7d26687148a1Tejun Heo * If there was a flush failure no regions can be marked clean. 854184153f9e483f9bb63339ed316e059962fe9794Mikulas Patocka */ 86d87f4c14f27dc82d215108d8392a7d26687148a1Tejun Heo int flush_failure; 874184153f9e483f9bb63339ed316e059962fe9794Mikulas Patocka 881f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen void *context; 891f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen sector_t target_begin; 901f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 911f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* Callback function to schedule bios writes */ 921f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen void (*dispatch_bios)(void *context, struct bio_list *bios); 931f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 941f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* Callback function to wakeup callers worker thread. */ 951f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen void (*wakeup_workers)(void *context); 961f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 971f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* Callback function to wakeup callers recovery waiters. */ 981f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen void (*wakeup_all_recovery_waiters)(void *context); 991f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen}; 1001f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1011f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstruct dm_region { 1021f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region_hash *rh; /* FIXME: can we get rid of this ? */ 1031f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen region_t key; 1041f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen int state; 1051f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1061f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct list_head hash_list; 1071f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct list_head list; 1081f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1091f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen atomic_t pending; 1101f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct bio_list delayed_bios; 1111f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen}; 1121f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1131f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen/* 1141f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Conversion fns 1151f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 1161f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstatic region_t dm_rh_sector_to_region(struct dm_region_hash *rh, sector_t sector) 1171f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 1181f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return sector >> rh->region_shift; 1191f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 1201f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1211f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagensector_t dm_rh_region_to_sector(struct dm_region_hash *rh, region_t region) 1221f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 1231f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return region << rh->region_shift; 1241f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 1251f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_region_to_sector); 1261f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1271f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenregion_t dm_rh_bio_to_region(struct dm_region_hash *rh, struct bio *bio) 1281f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 1291f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return dm_rh_sector_to_region(rh, bio->bi_sector - rh->target_begin); 1301f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 1311f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_bio_to_region); 1321f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1331f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenvoid *dm_rh_region_context(struct dm_region *reg) 1341f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 1351f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return reg->rh->context; 1361f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 1371f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_region_context); 1381f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1391f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenregion_t dm_rh_get_region_key(struct dm_region *reg) 1401f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 1411f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return reg->key; 1421f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 1431f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_get_region_key); 1441f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1451f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagensector_t dm_rh_get_region_size(struct dm_region_hash *rh) 1461f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 1471f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return rh->region_size; 1481f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 1491f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_get_region_size); 1501f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1511f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen/* 1521f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * FIXME: shall we pass in a structure instead of all these args to 1531f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * dm_region_hash_create()???? 1541f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 1551f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#define RH_HASH_MULT 2654435387U 1561f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#define RH_HASH_SHIFT 12 1571f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1581f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen#define MIN_REGIONS 64 1591f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstruct dm_region_hash *dm_region_hash_create( 1601f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen void *context, void (*dispatch_bios)(void *context, 1611f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct bio_list *bios), 1621f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen void (*wakeup_workers)(void *context), 1631f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen void (*wakeup_all_recovery_waiters)(void *context), 1641f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen sector_t target_begin, unsigned max_recovery, 1651f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_dirty_log *log, uint32_t region_size, 1661f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen region_t nr_regions) 1671f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 1681f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region_hash *rh; 1691f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen unsigned nr_buckets, max_buckets; 1701f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen size_t i; 1711f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1721f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* 1731f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Calculate a suitable number of buckets for our hash 1741f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * table. 1751f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 1761f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen max_buckets = nr_regions >> 6; 1771f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen for (nr_buckets = 128u; nr_buckets < max_buckets; nr_buckets <<= 1) 1781f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen ; 1791f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen nr_buckets >>= 1; 1801f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1811f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh = kmalloc(sizeof(*rh), GFP_KERNEL); 1821f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (!rh) { 1831f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen DMERR("unable to allocate region hash memory"); 1841f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return ERR_PTR(-ENOMEM); 1851f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 1861f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 1871f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->context = context; 1881f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->dispatch_bios = dispatch_bios; 1891f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->wakeup_workers = wakeup_workers; 1901f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->wakeup_all_recovery_waiters = wakeup_all_recovery_waiters; 1911f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->target_begin = target_begin; 1921f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->max_recovery = max_recovery; 1931f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->log = log; 1941f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->region_size = region_size; 1951f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->region_shift = ffs(region_size) - 1; 1961f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rwlock_init(&rh->hash_lock); 1971f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->mask = nr_buckets - 1; 1981f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->nr_buckets = nr_buckets; 1991f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2001f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->shift = RH_HASH_SHIFT; 2011f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->prime = RH_HASH_MULT; 2021f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2031f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->buckets = vmalloc(nr_buckets * sizeof(*rh->buckets)); 2041f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (!rh->buckets) { 2051f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen DMERR("unable to allocate region hash bucket memory"); 2061f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen kfree(rh); 2071f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return ERR_PTR(-ENOMEM); 2081f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 2091f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2101f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen for (i = 0; i < nr_buckets; i++) 2111f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen INIT_LIST_HEAD(rh->buckets + i); 2121f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2131f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_lock_init(&rh->region_lock); 2141f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen sema_init(&rh->recovery_count, 0); 2151f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen atomic_set(&rh->recovery_in_flight, 0); 2161f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen INIT_LIST_HEAD(&rh->clean_regions); 2171f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen INIT_LIST_HEAD(&rh->quiesced_regions); 2181f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen INIT_LIST_HEAD(&rh->recovered_regions); 2191f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen INIT_LIST_HEAD(&rh->failed_recovered_regions); 220d87f4c14f27dc82d215108d8392a7d26687148a1Tejun Heo rh->flush_failure = 0; 2211f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2221f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->region_pool = mempool_create_kmalloc_pool(MIN_REGIONS, 2231f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen sizeof(struct dm_region)); 2241f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (!rh->region_pool) { 2251f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen vfree(rh->buckets); 2261f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen kfree(rh); 2271f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh = ERR_PTR(-ENOMEM); 2281f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 2291f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2301f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return rh; 2311f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 2321f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_region_hash_create); 2331f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2341f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenvoid dm_region_hash_destroy(struct dm_region_hash *rh) 2351f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 2361f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen unsigned h; 2371f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg, *nreg; 2381f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2391f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen BUG_ON(!list_empty(&rh->quiesced_regions)); 2401f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen for (h = 0; h < rh->nr_buckets; h++) { 2411f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_for_each_entry_safe(reg, nreg, rh->buckets + h, 2421f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen hash_list) { 2431f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen BUG_ON(atomic_read(®->pending)); 2441f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen mempool_free(reg, rh->region_pool); 2451f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 2461f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 2471f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2481f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (rh->log) 2491f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen dm_dirty_log_destroy(rh->log); 2501f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2511f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (rh->region_pool) 2521f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen mempool_destroy(rh->region_pool); 2531f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2541f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen vfree(rh->buckets); 2551f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen kfree(rh); 2561f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 2571f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_region_hash_destroy); 2581f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2591f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstruct dm_dirty_log *dm_rh_dirty_log(struct dm_region_hash *rh) 2601f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 2611f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return rh->log; 2621f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 2631f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_dirty_log); 2641f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2651f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstatic unsigned rh_hash(struct dm_region_hash *rh, region_t region) 2661f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 2671f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return (unsigned) ((region * rh->prime) >> rh->shift) & rh->mask; 2681f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 2691f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2701f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstatic struct dm_region *__rh_lookup(struct dm_region_hash *rh, region_t region) 2711f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 2721f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg; 2731f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct list_head *bucket = rh->buckets + rh_hash(rh, region); 2741f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2751f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_for_each_entry(reg, bucket, hash_list) 2761f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (reg->key == region) 2771f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return reg; 2781f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2791f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return NULL; 2801f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 2811f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2821f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstatic void __rh_insert(struct dm_region_hash *rh, struct dm_region *reg) 2831f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 2841f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_add(®->hash_list, rh->buckets + rh_hash(rh, reg->key)); 2851f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 2861f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2871f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstatic struct dm_region *__rh_alloc(struct dm_region_hash *rh, region_t region) 2881f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 2891f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg, *nreg; 2901f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2911f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen nreg = mempool_alloc(rh->region_pool, GFP_ATOMIC); 2921f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (unlikely(!nreg)) 293a72986c562eeec3f7b992198c168f0f41606fe53Mikulas Patocka nreg = kmalloc(sizeof(*nreg), GFP_NOIO | __GFP_NOFAIL); 2941f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 2951f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen nreg->state = rh->log->type->in_sync(rh->log, region, 1) ? 2961f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen DM_RH_CLEAN : DM_RH_NOSYNC; 2971f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen nreg->rh = rh; 2981f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen nreg->key = region; 2991f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen INIT_LIST_HEAD(&nreg->list); 3001f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen atomic_set(&nreg->pending, 0); 3011f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen bio_list_init(&nreg->delayed_bios); 3021f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3031f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen write_lock_irq(&rh->hash_lock); 3041f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = __rh_lookup(rh, region); 3051f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (reg) 3061f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* We lost the race. */ 3071f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen mempool_free(nreg, rh->region_pool); 3081f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen else { 3091f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen __rh_insert(rh, nreg); 3101f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (nreg->state == DM_RH_CLEAN) { 3111f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_lock(&rh->region_lock); 3121f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_add(&nreg->list, &rh->clean_regions); 3131f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_unlock(&rh->region_lock); 3141f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 3151f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3161f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = nreg; 3171f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 3181f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen write_unlock_irq(&rh->hash_lock); 3191f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3201f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return reg; 3211f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 3221f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3231f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstatic struct dm_region *__rh_find(struct dm_region_hash *rh, region_t region) 3241f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 3251f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg; 3261f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3271f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = __rh_lookup(rh, region); 3281f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (!reg) { 3291f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_unlock(&rh->hash_lock); 3301f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = __rh_alloc(rh, region); 3311f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_lock(&rh->hash_lock); 3321f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 3331f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3341f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return reg; 3351f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 3361f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3371f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenint dm_rh_get_state(struct dm_region_hash *rh, region_t region, int may_block) 3381f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 3391f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen int r; 3401f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg; 3411f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3421f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_lock(&rh->hash_lock); 3431f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = __rh_lookup(rh, region); 3441f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_unlock(&rh->hash_lock); 3451f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3461f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (reg) 3471f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return reg->state; 3481f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3491f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* 3501f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * The region wasn't in the hash, so we fall back to the 3511f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * dirty log. 3521f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 3531f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen r = rh->log->type->in_sync(rh->log, region, may_block); 3541f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3551f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* 3561f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Any error from the dirty log (eg. -EWOULDBLOCK) gets 3571f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * taken as a DM_RH_NOSYNC 3581f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 3591f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return r == 1 ? DM_RH_CLEAN : DM_RH_NOSYNC; 3601f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 3611f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_get_state); 3621f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3631f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstatic void complete_resync_work(struct dm_region *reg, int success) 3641f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 3651f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region_hash *rh = reg->rh; 3661f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3671f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->log->type->set_region_sync(rh->log, reg->key, success); 3681f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3691f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* 3701f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Dispatch the bios before we call 'wake_up_all'. 3711f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * This is important because if we are suspending, 3721f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * we want to know that recovery is complete and 3731f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * the work queue is flushed. If we wake_up_all 3741f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * before we dispatch_bios (queue bios and call wake()), 3751f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * then we risk suspending before the work queue 3761f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * has been properly flushed. 3771f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 3781f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->dispatch_bios(rh->context, ®->delayed_bios); 3791f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (atomic_dec_and_test(&rh->recovery_in_flight)) 3801f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->wakeup_all_recovery_waiters(rh->context); 3811f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen up(&rh->recovery_count); 3821f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 3831f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 3841f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen/* dm_rh_mark_nosync 3851f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * @ms 3861f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * @bio 3871f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 3881f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * The bio was written on some mirror(s) but failed on other mirror(s). 3891f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * We can successfully endio the bio but should avoid the region being 3901f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * marked clean by setting the state DM_RH_NOSYNC. 3911f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 3921f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * This function is _not_ safe in interrupt context! 3931f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 394c58098be979509a54021e837a47fcad08db31f94Mikulas Patockavoid dm_rh_mark_nosync(struct dm_region_hash *rh, struct bio *bio) 3951f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 3961f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen unsigned long flags; 3971f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_dirty_log *log = rh->log; 3981f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg; 3991f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen region_t region = dm_rh_bio_to_region(rh, bio); 4001f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen int recovering = 0; 4011f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 402d87f4c14f27dc82d215108d8392a7d26687148a1Tejun Heo if (bio->bi_rw & REQ_FLUSH) { 403d87f4c14f27dc82d215108d8392a7d26687148a1Tejun Heo rh->flush_failure = 1; 4044184153f9e483f9bb63339ed316e059962fe9794Mikulas Patocka return; 4054184153f9e483f9bb63339ed316e059962fe9794Mikulas Patocka } 4064184153f9e483f9bb63339ed316e059962fe9794Mikulas Patocka 4071f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* We must inform the log that the sync count has changed. */ 4081f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen log->type->set_region_sync(log, region, 0); 4091f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4101f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_lock(&rh->hash_lock); 4111f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = __rh_find(rh, region); 4121f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_unlock(&rh->hash_lock); 4131f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4141f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* region hash entry should exist because write was in-flight */ 4151f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen BUG_ON(!reg); 4161f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen BUG_ON(!list_empty(®->list)); 4171f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4181f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_lock_irqsave(&rh->region_lock, flags); 4191f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* 4201f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Possible cases: 4211f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 1) DM_RH_DIRTY 42225985edcedea6396277003854657b5f3cb31a628Lucas De Marchi * 2) DM_RH_NOSYNC: was dirty, other preceding writes failed 4231f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 3) DM_RH_RECOVERING: flushing pending writes 4241f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Either case, the region should have not been connected to list. 4251f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 4261f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen recovering = (reg->state == DM_RH_RECOVERING); 4271f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg->state = DM_RH_NOSYNC; 4281f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen BUG_ON(!list_empty(®->list)); 4291f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_unlock_irqrestore(&rh->region_lock, flags); 4301f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4311f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (recovering) 4321f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen complete_resync_work(reg, 0); 4331f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 4341f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_mark_nosync); 4351f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4361f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenvoid dm_rh_update_states(struct dm_region_hash *rh, int errors_handled) 4371f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 4381f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg, *next; 4391f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4401f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen LIST_HEAD(clean); 4411f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen LIST_HEAD(recovered); 4421f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen LIST_HEAD(failed_recovered); 4431f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4441f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* 4451f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Quickly grab the lists. 4461f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 4471f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen write_lock_irq(&rh->hash_lock); 4481f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_lock(&rh->region_lock); 4491f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (!list_empty(&rh->clean_regions)) { 4501f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_splice_init(&rh->clean_regions, &clean); 4511f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4521f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_for_each_entry(reg, &clean, list) 4531f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_del(®->hash_list); 4541f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 4551f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4561f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (!list_empty(&rh->recovered_regions)) { 4571f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_splice_init(&rh->recovered_regions, &recovered); 4581f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4591f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_for_each_entry(reg, &recovered, list) 4601f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_del(®->hash_list); 4611f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 4621f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4631f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (!list_empty(&rh->failed_recovered_regions)) { 4641f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_splice_init(&rh->failed_recovered_regions, 4651f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen &failed_recovered); 4661f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4671f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_for_each_entry(reg, &failed_recovered, list) 4681f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_del(®->hash_list); 4691f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 4701f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4711f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_unlock(&rh->region_lock); 4721f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen write_unlock_irq(&rh->hash_lock); 4731f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4741f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* 4751f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * All the regions on the recovered and clean lists have 4761f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * now been pulled out of the system, so no need to do 4771f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * any more locking. 4781f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 4791f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_for_each_entry_safe(reg, next, &recovered, list) { 4801f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->log->type->clear_region(rh->log, reg->key); 4811f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen complete_resync_work(reg, 1); 4821f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen mempool_free(reg, rh->region_pool); 4831f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 4841f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4851f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_for_each_entry_safe(reg, next, &failed_recovered, list) { 4861f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen complete_resync_work(reg, errors_handled ? 0 : 1); 4871f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen mempool_free(reg, rh->region_pool); 4881f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 4891f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4901f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_for_each_entry_safe(reg, next, &clean, list) { 4911f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->log->type->clear_region(rh->log, reg->key); 4921f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen mempool_free(reg, rh->region_pool); 4931f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 4941f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4951f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->log->type->flush(rh->log); 4961f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 4971f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_update_states); 4981f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 4991f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstatic void rh_inc(struct dm_region_hash *rh, region_t region) 5001f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 5011f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg; 5021f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5031f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_lock(&rh->hash_lock); 5041f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = __rh_find(rh, region); 5051f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5061f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_lock_irq(&rh->region_lock); 5071f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen atomic_inc(®->pending); 5081f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5091f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (reg->state == DM_RH_CLEAN) { 5101f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg->state = DM_RH_DIRTY; 5111f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_del_init(®->list); /* take off the clean list */ 5121f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_unlock_irq(&rh->region_lock); 5131f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5141f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->log->type->mark_region(rh->log, reg->key); 5151f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } else 5161f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_unlock_irq(&rh->region_lock); 5171f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5181f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5191f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_unlock(&rh->hash_lock); 5201f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 5211f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5221f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenvoid dm_rh_inc_pending(struct dm_region_hash *rh, struct bio_list *bios) 5231f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 5241f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct bio *bio; 5251f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5264184153f9e483f9bb63339ed316e059962fe9794Mikulas Patocka for (bio = bios->head; bio; bio = bio->bi_next) { 527d87f4c14f27dc82d215108d8392a7d26687148a1Tejun Heo if (bio->bi_rw & REQ_FLUSH) 5284184153f9e483f9bb63339ed316e059962fe9794Mikulas Patocka continue; 5291f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh_inc(rh, dm_rh_bio_to_region(rh, bio)); 5304184153f9e483f9bb63339ed316e059962fe9794Mikulas Patocka } 5311f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 5321f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_inc_pending); 5331f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5341f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenvoid dm_rh_dec(struct dm_region_hash *rh, region_t region) 5351f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 5361f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen unsigned long flags; 5371f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg; 5381f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen int should_wake = 0; 5391f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5401f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_lock(&rh->hash_lock); 5411f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = __rh_lookup(rh, region); 5421f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_unlock(&rh->hash_lock); 5431f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5441f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_lock_irqsave(&rh->region_lock, flags); 5451f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (atomic_dec_and_test(®->pending)) { 5461f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* 5471f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * There is no pending I/O for this region. 5481f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * We can move the region to corresponding list for next action. 5491f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * At this point, the region is not yet connected to any list. 5501f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * 5511f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * If the state is DM_RH_NOSYNC, the region should be kept off 5521f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * from clean list. 5531f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * The hash entry for DM_RH_NOSYNC will remain in memory 5541f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * until the region is recovered or the map is reloaded. 5551f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 5561f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5571f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* do nothing for DM_RH_NOSYNC */ 558d87f4c14f27dc82d215108d8392a7d26687148a1Tejun Heo if (unlikely(rh->flush_failure)) { 5594184153f9e483f9bb63339ed316e059962fe9794Mikulas Patocka /* 560d87f4c14f27dc82d215108d8392a7d26687148a1Tejun Heo * If a write flush failed some time ago, we 5614184153f9e483f9bb63339ed316e059962fe9794Mikulas Patocka * don't know whether or not this write made it 5624184153f9e483f9bb63339ed316e059962fe9794Mikulas Patocka * to the disk, so we must resync the device. 5634184153f9e483f9bb63339ed316e059962fe9794Mikulas Patocka */ 5644184153f9e483f9bb63339ed316e059962fe9794Mikulas Patocka reg->state = DM_RH_NOSYNC; 5654184153f9e483f9bb63339ed316e059962fe9794Mikulas Patocka } else if (reg->state == DM_RH_RECOVERING) { 5661f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_add_tail(®->list, &rh->quiesced_regions); 5671f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } else if (reg->state == DM_RH_DIRTY) { 5681f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg->state = DM_RH_CLEAN; 5691f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_add(®->list, &rh->clean_regions); 5701f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 5711f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen should_wake = 1; 5721f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 5731f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_unlock_irqrestore(&rh->region_lock, flags); 5741f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5751f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (should_wake) 5761f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->wakeup_workers(rh->context); 5771f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 5781f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_dec); 5791f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5801f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen/* 5811f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Starts quiescing a region in preparation for recovery. 5821f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 5831f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstatic int __rh_recovery_prepare(struct dm_region_hash *rh) 5841f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 5851f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen int r; 5861f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen region_t region; 5871f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg; 5881f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5891f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* 5901f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Ask the dirty log what's next. 5911f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 5921f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen r = rh->log->type->get_resync_work(rh->log, ®ion); 5931f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (r <= 0) 5941f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return r; 5951f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 5961f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* 5971f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Get this region, and start it quiescing by setting the 5981f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * recovering flag. 5991f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 6001f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_lock(&rh->hash_lock); 6011f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = __rh_find(rh, region); 6021f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_unlock(&rh->hash_lock); 6031f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6041f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_lock_irq(&rh->region_lock); 6051f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg->state = DM_RH_RECOVERING; 6061f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6071f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* Already quiesced ? */ 6081f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (atomic_read(®->pending)) 6091f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_del_init(®->list); 6101f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen else 6111f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_move(®->list, &rh->quiesced_regions); 6121f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6131f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_unlock_irq(&rh->region_lock); 6141f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6151f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return 1; 6161f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 6171f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6181f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenvoid dm_rh_recovery_prepare(struct dm_region_hash *rh) 6191f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 6201f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* Extra reference to avoid race with dm_rh_stop_recovery */ 6211f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen atomic_inc(&rh->recovery_in_flight); 6221f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6231f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen while (!down_trylock(&rh->recovery_count)) { 6241f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen atomic_inc(&rh->recovery_in_flight); 6251f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (__rh_recovery_prepare(rh) <= 0) { 6261f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen atomic_dec(&rh->recovery_in_flight); 6271f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen up(&rh->recovery_count); 6281f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen break; 6291f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 6301f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 6311f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6321f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* Drop the extra reference */ 6331f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (atomic_dec_and_test(&rh->recovery_in_flight)) 6341f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->wakeup_all_recovery_waiters(rh->context); 6351f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 6361f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_recovery_prepare); 6371f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6381f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen/* 6391f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen * Returns any quiesced regions. 6401f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen */ 6411f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenstruct dm_region *dm_rh_recovery_start(struct dm_region_hash *rh) 6421f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 6431f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg = NULL; 6441f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6451f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_lock_irq(&rh->region_lock); 6461f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (!list_empty(&rh->quiesced_regions)) { 6471f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = list_entry(rh->quiesced_regions.next, 6481f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region, list); 6491f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_del_init(®->list); /* remove from the quiesced list */ 6501f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen } 6511f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_unlock_irq(&rh->region_lock); 6521f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6531f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return reg; 6541f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 6551f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_recovery_start); 6561f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6571f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenvoid dm_rh_recovery_end(struct dm_region *reg, int success) 6581f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 6591f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region_hash *rh = reg->rh; 6601f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6611f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_lock_irq(&rh->region_lock); 6621f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen if (success) 6631f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_add(®->list, ®->rh->recovered_regions); 664558569aa9d83e016295bac77d900342908d7fd85Takahiro Yasui else 6651f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen list_add(®->list, ®->rh->failed_recovered_regions); 666558569aa9d83e016295bac77d900342908d7fd85Takahiro Yasui 6671f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen spin_unlock_irq(&rh->region_lock); 6681f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6691f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->wakeup_workers(rh->context); 6701f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 6711f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_recovery_end); 6721f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6731f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen/* Return recovery in flight count. */ 6741f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenint dm_rh_recovery_in_flight(struct dm_region_hash *rh) 6751f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 6761f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return atomic_read(&rh->recovery_in_flight); 6771f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 6781f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_recovery_in_flight); 6791f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6801f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenint dm_rh_flush(struct dm_region_hash *rh) 6811f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 6821f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen return rh->log->type->flush(rh->log); 6831f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 6841f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_flush); 6851f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6861f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenvoid dm_rh_delay(struct dm_region_hash *rh, struct bio *bio) 6871f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 6881f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen struct dm_region *reg; 6891f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6901f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_lock(&rh->hash_lock); 6911f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen reg = __rh_find(rh, dm_rh_bio_to_region(rh, bio)); 6921f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen bio_list_add(®->delayed_bios, bio); 6931f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen read_unlock(&rh->hash_lock); 6941f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 6951f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_delay); 6961f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 6971f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenvoid dm_rh_stop_recovery(struct dm_region_hash *rh) 6981f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 6991f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen int i; 7001f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 7011f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen /* wait for any recovering regions */ 7021f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen for (i = 0; i < rh->max_recovery; i++) 7031f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen down(&rh->recovery_count); 7041f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 7051f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_stop_recovery); 7061f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 7071f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagenvoid dm_rh_start_recovery(struct dm_region_hash *rh) 7081f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen{ 7091f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen int i; 7101f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 7111f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen for (i = 0; i < rh->max_recovery; i++) 7121f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen up(&rh->recovery_count); 7131f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 7141f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen rh->wakeup_workers(rh->context); 7151f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen} 7161f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenEXPORT_SYMBOL_GPL(dm_rh_start_recovery); 7171f965b19437017cea6d3f3f46acdc5acae5fd011Heinz Mauelshagen 7181f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenMODULE_DESCRIPTION(DM_NAME " region hash"); 7191f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenMODULE_AUTHOR("Joe Thornber/Heinz Mauelshagen <dm-devel@redhat.com>"); 7201f965b19437017cea6d3f3f46acdc5acae5fd011Heinz MauelshagenMODULE_LICENSE("GPL"); 721