raid5.c revision 417b8d4ac868cf58d6c68f52d72f7648413e0edc
1/* 2 * raid5.c : Multiple Devices driver for Linux 3 * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman 4 * Copyright (C) 1999, 2000 Ingo Molnar 5 * Copyright (C) 2002, 2003 H. Peter Anvin 6 * 7 * RAID-4/5/6 management functions. 8 * Thanks to Penguin Computing for making the RAID-6 development possible 9 * by donating a test server! 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2, or (at your option) 14 * any later version. 15 * 16 * You should have received a copy of the GNU General Public License 17 * (for example /usr/src/linux/COPYING); if not, write to the Free 18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 19 */ 20 21/* 22 * BITMAP UNPLUGGING: 23 * 24 * The sequencing for updating the bitmap reliably is a little 25 * subtle (and I got it wrong the first time) so it deserves some 26 * explanation. 27 * 28 * We group bitmap updates into batches. Each batch has a number. 29 * We may write out several batches at once, but that isn't very important. 30 * conf->bm_write is the number of the last batch successfully written. 31 * conf->bm_flush is the number of the last batch that was closed to 32 * new additions. 33 * When we discover that we will need to write to any block in a stripe 34 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq 35 * the number of the batch it will be in. This is bm_flush+1. 36 * When we are ready to do a write, if that batch hasn't been written yet, 37 * we plug the array and queue the stripe for later. 38 * When an unplug happens, we increment bm_flush, thus closing the current 39 * batch. 40 * When we notice that bm_flush > bm_write, we write out all pending updates 41 * to the bitmap, and advance bm_write to where bm_flush was. 42 * This may occasionally write a bit out twice, but is sure never to 43 * miss any bits. 44 */ 45 46#include <linux/blkdev.h> 47#include <linux/kthread.h> 48#include <linux/raid/pq.h> 49#include <linux/async_tx.h> 50#include <linux/async.h> 51#include <linux/seq_file.h> 52#include <linux/cpu.h> 53#include "md.h" 54#include "raid5.h" 55#include "bitmap.h" 56 57/* 58 * Stripe cache 59 */ 60 61#define NR_STRIPES 256 62#define STRIPE_SIZE PAGE_SIZE 63#define STRIPE_SHIFT (PAGE_SHIFT - 9) 64#define STRIPE_SECTORS (STRIPE_SIZE>>9) 65#define IO_THRESHOLD 1 66#define BYPASS_THRESHOLD 1 67#define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head)) 68#define HASH_MASK (NR_HASH - 1) 69 70#define stripe_hash(conf, sect) (&((conf)->stripe_hashtbl[((sect) >> STRIPE_SHIFT) & HASH_MASK])) 71 72/* bio's attached to a stripe+device for I/O are linked together in bi_sector 73 * order without overlap. There may be several bio's per stripe+device, and 74 * a bio could span several devices. 75 * When walking this list for a particular stripe+device, we must never proceed 76 * beyond a bio that extends past this device, as the next bio might no longer 77 * be valid. 78 * This macro is used to determine the 'next' bio in the list, given the sector 79 * of the current stripe+device 80 */ 81#define r5_next_bio(bio, sect) ( ( (bio)->bi_sector + ((bio)->bi_size>>9) < sect + STRIPE_SECTORS) ? (bio)->bi_next : NULL) 82/* 83 * The following can be used to debug the driver 84 */ 85#define RAID5_PARANOIA 1 86#if RAID5_PARANOIA && defined(CONFIG_SMP) 87# define CHECK_DEVLOCK() assert_spin_locked(&conf->device_lock) 88#else 89# define CHECK_DEVLOCK() 90#endif 91 92#ifdef DEBUG 93#define inline 94#define __inline__ 95#endif 96 97#define printk_rl(args...) ((void) (printk_ratelimit() && printk(args))) 98 99/* 100 * We maintain a biased count of active stripes in the bottom 16 bits of 101 * bi_phys_segments, and a count of processed stripes in the upper 16 bits 102 */ 103static inline int raid5_bi_phys_segments(struct bio *bio) 104{ 105 return bio->bi_phys_segments & 0xffff; 106} 107 108static inline int raid5_bi_hw_segments(struct bio *bio) 109{ 110 return (bio->bi_phys_segments >> 16) & 0xffff; 111} 112 113static inline int raid5_dec_bi_phys_segments(struct bio *bio) 114{ 115 --bio->bi_phys_segments; 116 return raid5_bi_phys_segments(bio); 117} 118 119static inline int raid5_dec_bi_hw_segments(struct bio *bio) 120{ 121 unsigned short val = raid5_bi_hw_segments(bio); 122 123 --val; 124 bio->bi_phys_segments = (val << 16) | raid5_bi_phys_segments(bio); 125 return val; 126} 127 128static inline void raid5_set_bi_hw_segments(struct bio *bio, unsigned int cnt) 129{ 130 bio->bi_phys_segments = raid5_bi_phys_segments(bio) || (cnt << 16); 131} 132 133/* Find first data disk in a raid6 stripe */ 134static inline int raid6_d0(struct stripe_head *sh) 135{ 136 if (sh->ddf_layout) 137 /* ddf always start from first device */ 138 return 0; 139 /* md starts just after Q block */ 140 if (sh->qd_idx == sh->disks - 1) 141 return 0; 142 else 143 return sh->qd_idx + 1; 144} 145static inline int raid6_next_disk(int disk, int raid_disks) 146{ 147 disk++; 148 return (disk < raid_disks) ? disk : 0; 149} 150 151/* When walking through the disks in a raid5, starting at raid6_d0, 152 * We need to map each disk to a 'slot', where the data disks are slot 153 * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk 154 * is raid_disks-1. This help does that mapping. 155 */ 156static int raid6_idx_to_slot(int idx, struct stripe_head *sh, 157 int *count, int syndrome_disks) 158{ 159 int slot; 160 161 if (idx == sh->pd_idx) 162 return syndrome_disks; 163 if (idx == sh->qd_idx) 164 return syndrome_disks + 1; 165 slot = (*count)++; 166 return slot; 167} 168 169static void return_io(struct bio *return_bi) 170{ 171 struct bio *bi = return_bi; 172 while (bi) { 173 174 return_bi = bi->bi_next; 175 bi->bi_next = NULL; 176 bi->bi_size = 0; 177 bio_endio(bi, 0); 178 bi = return_bi; 179 } 180} 181 182static void print_raid5_conf (raid5_conf_t *conf); 183 184static int stripe_operations_active(struct stripe_head *sh) 185{ 186 return sh->check_state || sh->reconstruct_state || 187 test_bit(STRIPE_BIOFILL_RUN, &sh->state) || 188 test_bit(STRIPE_COMPUTE_RUN, &sh->state); 189} 190 191static void __release_stripe(raid5_conf_t *conf, struct stripe_head *sh) 192{ 193 if (atomic_dec_and_test(&sh->count)) { 194 BUG_ON(!list_empty(&sh->lru)); 195 BUG_ON(atomic_read(&conf->active_stripes)==0); 196 if (test_bit(STRIPE_HANDLE, &sh->state)) { 197 if (test_bit(STRIPE_DELAYED, &sh->state)) { 198 list_add_tail(&sh->lru, &conf->delayed_list); 199 blk_plug_device(conf->mddev->queue); 200 } else if (test_bit(STRIPE_BIT_DELAY, &sh->state) && 201 sh->bm_seq - conf->seq_write > 0) { 202 list_add_tail(&sh->lru, &conf->bitmap_list); 203 blk_plug_device(conf->mddev->queue); 204 } else { 205 clear_bit(STRIPE_BIT_DELAY, &sh->state); 206 list_add_tail(&sh->lru, &conf->handle_list); 207 } 208 md_wakeup_thread(conf->mddev->thread); 209 } else { 210 BUG_ON(stripe_operations_active(sh)); 211 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 212 atomic_dec(&conf->preread_active_stripes); 213 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) 214 md_wakeup_thread(conf->mddev->thread); 215 } 216 atomic_dec(&conf->active_stripes); 217 if (!test_bit(STRIPE_EXPANDING, &sh->state)) { 218 list_add_tail(&sh->lru, &conf->inactive_list); 219 wake_up(&conf->wait_for_stripe); 220 if (conf->retry_read_aligned) 221 md_wakeup_thread(conf->mddev->thread); 222 } 223 } 224 } 225} 226 227static void release_stripe(struct stripe_head *sh) 228{ 229 raid5_conf_t *conf = sh->raid_conf; 230 unsigned long flags; 231 232 spin_lock_irqsave(&conf->device_lock, flags); 233 __release_stripe(conf, sh); 234 spin_unlock_irqrestore(&conf->device_lock, flags); 235} 236 237static inline void remove_hash(struct stripe_head *sh) 238{ 239 pr_debug("remove_hash(), stripe %llu\n", 240 (unsigned long long)sh->sector); 241 242 hlist_del_init(&sh->hash); 243} 244 245static inline void insert_hash(raid5_conf_t *conf, struct stripe_head *sh) 246{ 247 struct hlist_head *hp = stripe_hash(conf, sh->sector); 248 249 pr_debug("insert_hash(), stripe %llu\n", 250 (unsigned long long)sh->sector); 251 252 CHECK_DEVLOCK(); 253 hlist_add_head(&sh->hash, hp); 254} 255 256 257/* find an idle stripe, make sure it is unhashed, and return it. */ 258static struct stripe_head *get_free_stripe(raid5_conf_t *conf) 259{ 260 struct stripe_head *sh = NULL; 261 struct list_head *first; 262 263 CHECK_DEVLOCK(); 264 if (list_empty(&conf->inactive_list)) 265 goto out; 266 first = conf->inactive_list.next; 267 sh = list_entry(first, struct stripe_head, lru); 268 list_del_init(first); 269 remove_hash(sh); 270 atomic_inc(&conf->active_stripes); 271out: 272 return sh; 273} 274 275static void shrink_buffers(struct stripe_head *sh, int num) 276{ 277 struct page *p; 278 int i; 279 280 for (i=0; i<num ; i++) { 281 p = sh->dev[i].page; 282 if (!p) 283 continue; 284 sh->dev[i].page = NULL; 285 put_page(p); 286 } 287} 288 289static int grow_buffers(struct stripe_head *sh, int num) 290{ 291 int i; 292 293 for (i=0; i<num; i++) { 294 struct page *page; 295 296 if (!(page = alloc_page(GFP_KERNEL))) { 297 return 1; 298 } 299 sh->dev[i].page = page; 300 } 301 return 0; 302} 303 304static void raid5_build_block(struct stripe_head *sh, int i, int previous); 305static void stripe_set_idx(sector_t stripe, raid5_conf_t *conf, int previous, 306 struct stripe_head *sh); 307 308static void init_stripe(struct stripe_head *sh, sector_t sector, int previous) 309{ 310 raid5_conf_t *conf = sh->raid_conf; 311 int i; 312 313 BUG_ON(atomic_read(&sh->count) != 0); 314 BUG_ON(test_bit(STRIPE_HANDLE, &sh->state)); 315 BUG_ON(stripe_operations_active(sh)); 316 317 CHECK_DEVLOCK(); 318 pr_debug("init_stripe called, stripe %llu\n", 319 (unsigned long long)sh->sector); 320 321 remove_hash(sh); 322 323 sh->generation = conf->generation - previous; 324 sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks; 325 sh->sector = sector; 326 stripe_set_idx(sector, conf, previous, sh); 327 sh->state = 0; 328 329 330 for (i = sh->disks; i--; ) { 331 struct r5dev *dev = &sh->dev[i]; 332 333 if (dev->toread || dev->read || dev->towrite || dev->written || 334 test_bit(R5_LOCKED, &dev->flags)) { 335 printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n", 336 (unsigned long long)sh->sector, i, dev->toread, 337 dev->read, dev->towrite, dev->written, 338 test_bit(R5_LOCKED, &dev->flags)); 339 BUG(); 340 } 341 dev->flags = 0; 342 raid5_build_block(sh, i, previous); 343 } 344 insert_hash(conf, sh); 345} 346 347static struct stripe_head *__find_stripe(raid5_conf_t *conf, sector_t sector, 348 short generation) 349{ 350 struct stripe_head *sh; 351 struct hlist_node *hn; 352 353 CHECK_DEVLOCK(); 354 pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector); 355 hlist_for_each_entry(sh, hn, stripe_hash(conf, sector), hash) 356 if (sh->sector == sector && sh->generation == generation) 357 return sh; 358 pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector); 359 return NULL; 360} 361 362static void unplug_slaves(mddev_t *mddev); 363static void raid5_unplug_device(struct request_queue *q); 364 365static struct stripe_head * 366get_active_stripe(raid5_conf_t *conf, sector_t sector, 367 int previous, int noblock, int noquiesce) 368{ 369 struct stripe_head *sh; 370 371 pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector); 372 373 spin_lock_irq(&conf->device_lock); 374 375 do { 376 wait_event_lock_irq(conf->wait_for_stripe, 377 conf->quiesce == 0 || noquiesce, 378 conf->device_lock, /* nothing */); 379 sh = __find_stripe(conf, sector, conf->generation - previous); 380 if (!sh) { 381 if (!conf->inactive_blocked) 382 sh = get_free_stripe(conf); 383 if (noblock && sh == NULL) 384 break; 385 if (!sh) { 386 conf->inactive_blocked = 1; 387 wait_event_lock_irq(conf->wait_for_stripe, 388 !list_empty(&conf->inactive_list) && 389 (atomic_read(&conf->active_stripes) 390 < (conf->max_nr_stripes *3/4) 391 || !conf->inactive_blocked), 392 conf->device_lock, 393 raid5_unplug_device(conf->mddev->queue) 394 ); 395 conf->inactive_blocked = 0; 396 } else 397 init_stripe(sh, sector, previous); 398 } else { 399 if (atomic_read(&sh->count)) { 400 BUG_ON(!list_empty(&sh->lru) 401 && !test_bit(STRIPE_EXPANDING, &sh->state)); 402 } else { 403 if (!test_bit(STRIPE_HANDLE, &sh->state)) 404 atomic_inc(&conf->active_stripes); 405 if (list_empty(&sh->lru) && 406 !test_bit(STRIPE_EXPANDING, &sh->state)) 407 BUG(); 408 list_del_init(&sh->lru); 409 } 410 } 411 } while (sh == NULL); 412 413 if (sh) 414 atomic_inc(&sh->count); 415 416 spin_unlock_irq(&conf->device_lock); 417 return sh; 418} 419 420static void 421raid5_end_read_request(struct bio *bi, int error); 422static void 423raid5_end_write_request(struct bio *bi, int error); 424 425static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s) 426{ 427 raid5_conf_t *conf = sh->raid_conf; 428 int i, disks = sh->disks; 429 430 might_sleep(); 431 432 for (i = disks; i--; ) { 433 int rw; 434 struct bio *bi; 435 mdk_rdev_t *rdev; 436 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) 437 rw = WRITE; 438 else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags)) 439 rw = READ; 440 else 441 continue; 442 443 bi = &sh->dev[i].req; 444 445 bi->bi_rw = rw; 446 if (rw == WRITE) 447 bi->bi_end_io = raid5_end_write_request; 448 else 449 bi->bi_end_io = raid5_end_read_request; 450 451 rcu_read_lock(); 452 rdev = rcu_dereference(conf->disks[i].rdev); 453 if (rdev && test_bit(Faulty, &rdev->flags)) 454 rdev = NULL; 455 if (rdev) 456 atomic_inc(&rdev->nr_pending); 457 rcu_read_unlock(); 458 459 if (rdev) { 460 if (s->syncing || s->expanding || s->expanded) 461 md_sync_acct(rdev->bdev, STRIPE_SECTORS); 462 463 set_bit(STRIPE_IO_STARTED, &sh->state); 464 465 bi->bi_bdev = rdev->bdev; 466 pr_debug("%s: for %llu schedule op %ld on disc %d\n", 467 __func__, (unsigned long long)sh->sector, 468 bi->bi_rw, i); 469 atomic_inc(&sh->count); 470 bi->bi_sector = sh->sector + rdev->data_offset; 471 bi->bi_flags = 1 << BIO_UPTODATE; 472 bi->bi_vcnt = 1; 473 bi->bi_max_vecs = 1; 474 bi->bi_idx = 0; 475 bi->bi_io_vec = &sh->dev[i].vec; 476 bi->bi_io_vec[0].bv_len = STRIPE_SIZE; 477 bi->bi_io_vec[0].bv_offset = 0; 478 bi->bi_size = STRIPE_SIZE; 479 bi->bi_next = NULL; 480 if (rw == WRITE && 481 test_bit(R5_ReWrite, &sh->dev[i].flags)) 482 atomic_add(STRIPE_SECTORS, 483 &rdev->corrected_errors); 484 generic_make_request(bi); 485 } else { 486 if (rw == WRITE) 487 set_bit(STRIPE_DEGRADED, &sh->state); 488 pr_debug("skip op %ld on disc %d for sector %llu\n", 489 bi->bi_rw, i, (unsigned long long)sh->sector); 490 clear_bit(R5_LOCKED, &sh->dev[i].flags); 491 set_bit(STRIPE_HANDLE, &sh->state); 492 } 493 } 494} 495 496static struct dma_async_tx_descriptor * 497async_copy_data(int frombio, struct bio *bio, struct page *page, 498 sector_t sector, struct dma_async_tx_descriptor *tx) 499{ 500 struct bio_vec *bvl; 501 struct page *bio_page; 502 int i; 503 int page_offset; 504 struct async_submit_ctl submit; 505 enum async_tx_flags flags = 0; 506 507 if (bio->bi_sector >= sector) 508 page_offset = (signed)(bio->bi_sector - sector) * 512; 509 else 510 page_offset = (signed)(sector - bio->bi_sector) * -512; 511 512 if (frombio) 513 flags |= ASYNC_TX_FENCE; 514 init_async_submit(&submit, flags, tx, NULL, NULL, NULL); 515 516 bio_for_each_segment(bvl, bio, i) { 517 int len = bio_iovec_idx(bio, i)->bv_len; 518 int clen; 519 int b_offset = 0; 520 521 if (page_offset < 0) { 522 b_offset = -page_offset; 523 page_offset += b_offset; 524 len -= b_offset; 525 } 526 527 if (len > 0 && page_offset + len > STRIPE_SIZE) 528 clen = STRIPE_SIZE - page_offset; 529 else 530 clen = len; 531 532 if (clen > 0) { 533 b_offset += bio_iovec_idx(bio, i)->bv_offset; 534 bio_page = bio_iovec_idx(bio, i)->bv_page; 535 if (frombio) 536 tx = async_memcpy(page, bio_page, page_offset, 537 b_offset, clen, &submit); 538 else 539 tx = async_memcpy(bio_page, page, b_offset, 540 page_offset, clen, &submit); 541 } 542 /* chain the operations */ 543 submit.depend_tx = tx; 544 545 if (clen < len) /* hit end of page */ 546 break; 547 page_offset += len; 548 } 549 550 return tx; 551} 552 553static void ops_complete_biofill(void *stripe_head_ref) 554{ 555 struct stripe_head *sh = stripe_head_ref; 556 struct bio *return_bi = NULL; 557 raid5_conf_t *conf = sh->raid_conf; 558 int i; 559 560 pr_debug("%s: stripe %llu\n", __func__, 561 (unsigned long long)sh->sector); 562 563 /* clear completed biofills */ 564 spin_lock_irq(&conf->device_lock); 565 for (i = sh->disks; i--; ) { 566 struct r5dev *dev = &sh->dev[i]; 567 568 /* acknowledge completion of a biofill operation */ 569 /* and check if we need to reply to a read request, 570 * new R5_Wantfill requests are held off until 571 * !STRIPE_BIOFILL_RUN 572 */ 573 if (test_and_clear_bit(R5_Wantfill, &dev->flags)) { 574 struct bio *rbi, *rbi2; 575 576 BUG_ON(!dev->read); 577 rbi = dev->read; 578 dev->read = NULL; 579 while (rbi && rbi->bi_sector < 580 dev->sector + STRIPE_SECTORS) { 581 rbi2 = r5_next_bio(rbi, dev->sector); 582 if (!raid5_dec_bi_phys_segments(rbi)) { 583 rbi->bi_next = return_bi; 584 return_bi = rbi; 585 } 586 rbi = rbi2; 587 } 588 } 589 } 590 spin_unlock_irq(&conf->device_lock); 591 clear_bit(STRIPE_BIOFILL_RUN, &sh->state); 592 593 return_io(return_bi); 594 595 set_bit(STRIPE_HANDLE, &sh->state); 596 release_stripe(sh); 597} 598 599static void ops_run_biofill(struct stripe_head *sh) 600{ 601 struct dma_async_tx_descriptor *tx = NULL; 602 raid5_conf_t *conf = sh->raid_conf; 603 struct async_submit_ctl submit; 604 int i; 605 606 pr_debug("%s: stripe %llu\n", __func__, 607 (unsigned long long)sh->sector); 608 609 for (i = sh->disks; i--; ) { 610 struct r5dev *dev = &sh->dev[i]; 611 if (test_bit(R5_Wantfill, &dev->flags)) { 612 struct bio *rbi; 613 spin_lock_irq(&conf->device_lock); 614 dev->read = rbi = dev->toread; 615 dev->toread = NULL; 616 spin_unlock_irq(&conf->device_lock); 617 while (rbi && rbi->bi_sector < 618 dev->sector + STRIPE_SECTORS) { 619 tx = async_copy_data(0, rbi, dev->page, 620 dev->sector, tx); 621 rbi = r5_next_bio(rbi, dev->sector); 622 } 623 } 624 } 625 626 atomic_inc(&sh->count); 627 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL); 628 async_trigger_callback(&submit); 629} 630 631static void mark_target_uptodate(struct stripe_head *sh, int target) 632{ 633 struct r5dev *tgt; 634 635 if (target < 0) 636 return; 637 638 tgt = &sh->dev[target]; 639 set_bit(R5_UPTODATE, &tgt->flags); 640 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 641 clear_bit(R5_Wantcompute, &tgt->flags); 642} 643 644static void ops_complete_compute(void *stripe_head_ref) 645{ 646 struct stripe_head *sh = stripe_head_ref; 647 648 pr_debug("%s: stripe %llu\n", __func__, 649 (unsigned long long)sh->sector); 650 651 /* mark the computed target(s) as uptodate */ 652 mark_target_uptodate(sh, sh->ops.target); 653 mark_target_uptodate(sh, sh->ops.target2); 654 655 clear_bit(STRIPE_COMPUTE_RUN, &sh->state); 656 if (sh->check_state == check_state_compute_run) 657 sh->check_state = check_state_compute_result; 658 set_bit(STRIPE_HANDLE, &sh->state); 659 release_stripe(sh); 660} 661 662/* return a pointer to the address conversion region of the scribble buffer */ 663static addr_conv_t *to_addr_conv(struct stripe_head *sh, 664 struct raid5_percpu *percpu) 665{ 666 return percpu->scribble + sizeof(struct page *) * (sh->disks + 2); 667} 668 669static struct dma_async_tx_descriptor * 670ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu) 671{ 672 int disks = sh->disks; 673 struct page **xor_srcs = percpu->scribble; 674 int target = sh->ops.target; 675 struct r5dev *tgt = &sh->dev[target]; 676 struct page *xor_dest = tgt->page; 677 int count = 0; 678 struct dma_async_tx_descriptor *tx; 679 struct async_submit_ctl submit; 680 int i; 681 682 pr_debug("%s: stripe %llu block: %d\n", 683 __func__, (unsigned long long)sh->sector, target); 684 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 685 686 for (i = disks; i--; ) 687 if (i != target) 688 xor_srcs[count++] = sh->dev[i].page; 689 690 atomic_inc(&sh->count); 691 692 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL, 693 ops_complete_compute, sh, to_addr_conv(sh, percpu)); 694 if (unlikely(count == 1)) 695 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); 696 else 697 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 698 699 return tx; 700} 701 702/* set_syndrome_sources - populate source buffers for gen_syndrome 703 * @srcs - (struct page *) array of size sh->disks 704 * @sh - stripe_head to parse 705 * 706 * Populates srcs in proper layout order for the stripe and returns the 707 * 'count' of sources to be used in a call to async_gen_syndrome. The P 708 * destination buffer is recorded in srcs[count] and the Q destination 709 * is recorded in srcs[count+1]]. 710 */ 711static int set_syndrome_sources(struct page **srcs, struct stripe_head *sh) 712{ 713 int disks = sh->disks; 714 int syndrome_disks = sh->ddf_layout ? disks : (disks - 2); 715 int d0_idx = raid6_d0(sh); 716 int count; 717 int i; 718 719 for (i = 0; i < disks; i++) 720 srcs[i] = (void *)raid6_empty_zero_page; 721 722 count = 0; 723 i = d0_idx; 724 do { 725 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); 726 727 srcs[slot] = sh->dev[i].page; 728 i = raid6_next_disk(i, disks); 729 } while (i != d0_idx); 730 BUG_ON(count != syndrome_disks); 731 732 return count; 733} 734 735static struct dma_async_tx_descriptor * 736ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu) 737{ 738 int disks = sh->disks; 739 struct page **blocks = percpu->scribble; 740 int target; 741 int qd_idx = sh->qd_idx; 742 struct dma_async_tx_descriptor *tx; 743 struct async_submit_ctl submit; 744 struct r5dev *tgt; 745 struct page *dest; 746 int i; 747 int count; 748 749 if (sh->ops.target < 0) 750 target = sh->ops.target2; 751 else if (sh->ops.target2 < 0) 752 target = sh->ops.target; 753 else 754 /* we should only have one valid target */ 755 BUG(); 756 BUG_ON(target < 0); 757 pr_debug("%s: stripe %llu block: %d\n", 758 __func__, (unsigned long long)sh->sector, target); 759 760 tgt = &sh->dev[target]; 761 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 762 dest = tgt->page; 763 764 atomic_inc(&sh->count); 765 766 if (target == qd_idx) { 767 count = set_syndrome_sources(blocks, sh); 768 blocks[count] = NULL; /* regenerating p is not necessary */ 769 BUG_ON(blocks[count+1] != dest); /* q should already be set */ 770 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 771 ops_complete_compute, sh, 772 to_addr_conv(sh, percpu)); 773 tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); 774 } else { 775 /* Compute any data- or p-drive using XOR */ 776 count = 0; 777 for (i = disks; i-- ; ) { 778 if (i == target || i == qd_idx) 779 continue; 780 blocks[count++] = sh->dev[i].page; 781 } 782 783 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, 784 NULL, ops_complete_compute, sh, 785 to_addr_conv(sh, percpu)); 786 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit); 787 } 788 789 return tx; 790} 791 792static struct dma_async_tx_descriptor * 793ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu) 794{ 795 int i, count, disks = sh->disks; 796 int syndrome_disks = sh->ddf_layout ? disks : disks-2; 797 int d0_idx = raid6_d0(sh); 798 int faila = -1, failb = -1; 799 int target = sh->ops.target; 800 int target2 = sh->ops.target2; 801 struct r5dev *tgt = &sh->dev[target]; 802 struct r5dev *tgt2 = &sh->dev[target2]; 803 struct dma_async_tx_descriptor *tx; 804 struct page **blocks = percpu->scribble; 805 struct async_submit_ctl submit; 806 807 pr_debug("%s: stripe %llu block1: %d block2: %d\n", 808 __func__, (unsigned long long)sh->sector, target, target2); 809 BUG_ON(target < 0 || target2 < 0); 810 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 811 BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags)); 812 813 /* we need to open-code set_syndrome_sources to handle the 814 * slot number conversion for 'faila' and 'failb' 815 */ 816 for (i = 0; i < disks ; i++) 817 blocks[i] = (void *)raid6_empty_zero_page; 818 count = 0; 819 i = d0_idx; 820 do { 821 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); 822 823 blocks[slot] = sh->dev[i].page; 824 825 if (i == target) 826 faila = slot; 827 if (i == target2) 828 failb = slot; 829 i = raid6_next_disk(i, disks); 830 } while (i != d0_idx); 831 BUG_ON(count != syndrome_disks); 832 833 BUG_ON(faila == failb); 834 if (failb < faila) 835 swap(faila, failb); 836 pr_debug("%s: stripe: %llu faila: %d failb: %d\n", 837 __func__, (unsigned long long)sh->sector, faila, failb); 838 839 atomic_inc(&sh->count); 840 841 if (failb == syndrome_disks+1) { 842 /* Q disk is one of the missing disks */ 843 if (faila == syndrome_disks) { 844 /* Missing P+Q, just recompute */ 845 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 846 ops_complete_compute, sh, 847 to_addr_conv(sh, percpu)); 848 return async_gen_syndrome(blocks, 0, count+2, 849 STRIPE_SIZE, &submit); 850 } else { 851 struct page *dest; 852 int data_target; 853 int qd_idx = sh->qd_idx; 854 855 /* Missing D+Q: recompute D from P, then recompute Q */ 856 if (target == qd_idx) 857 data_target = target2; 858 else 859 data_target = target; 860 861 count = 0; 862 for (i = disks; i-- ; ) { 863 if (i == data_target || i == qd_idx) 864 continue; 865 blocks[count++] = sh->dev[i].page; 866 } 867 dest = sh->dev[data_target].page; 868 init_async_submit(&submit, 869 ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, 870 NULL, NULL, NULL, 871 to_addr_conv(sh, percpu)); 872 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, 873 &submit); 874 875 count = set_syndrome_sources(blocks, sh); 876 init_async_submit(&submit, ASYNC_TX_FENCE, tx, 877 ops_complete_compute, sh, 878 to_addr_conv(sh, percpu)); 879 return async_gen_syndrome(blocks, 0, count+2, 880 STRIPE_SIZE, &submit); 881 } 882 } else { 883 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 884 ops_complete_compute, sh, 885 to_addr_conv(sh, percpu)); 886 if (failb == syndrome_disks) { 887 /* We're missing D+P. */ 888 return async_raid6_datap_recov(syndrome_disks+2, 889 STRIPE_SIZE, faila, 890 blocks, &submit); 891 } else { 892 /* We're missing D+D. */ 893 return async_raid6_2data_recov(syndrome_disks+2, 894 STRIPE_SIZE, faila, failb, 895 blocks, &submit); 896 } 897 } 898} 899 900 901static void ops_complete_prexor(void *stripe_head_ref) 902{ 903 struct stripe_head *sh = stripe_head_ref; 904 905 pr_debug("%s: stripe %llu\n", __func__, 906 (unsigned long long)sh->sector); 907} 908 909static struct dma_async_tx_descriptor * 910ops_run_prexor(struct stripe_head *sh, struct raid5_percpu *percpu, 911 struct dma_async_tx_descriptor *tx) 912{ 913 int disks = sh->disks; 914 struct page **xor_srcs = percpu->scribble; 915 int count = 0, pd_idx = sh->pd_idx, i; 916 struct async_submit_ctl submit; 917 918 /* existing parity data subtracted */ 919 struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; 920 921 pr_debug("%s: stripe %llu\n", __func__, 922 (unsigned long long)sh->sector); 923 924 for (i = disks; i--; ) { 925 struct r5dev *dev = &sh->dev[i]; 926 /* Only process blocks that are known to be uptodate */ 927 if (test_bit(R5_Wantdrain, &dev->flags)) 928 xor_srcs[count++] = dev->page; 929 } 930 931 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, 932 ops_complete_prexor, sh, to_addr_conv(sh, percpu)); 933 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 934 935 return tx; 936} 937 938static struct dma_async_tx_descriptor * 939ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx) 940{ 941 int disks = sh->disks; 942 int i; 943 944 pr_debug("%s: stripe %llu\n", __func__, 945 (unsigned long long)sh->sector); 946 947 for (i = disks; i--; ) { 948 struct r5dev *dev = &sh->dev[i]; 949 struct bio *chosen; 950 951 if (test_and_clear_bit(R5_Wantdrain, &dev->flags)) { 952 struct bio *wbi; 953 954 spin_lock(&sh->lock); 955 chosen = dev->towrite; 956 dev->towrite = NULL; 957 BUG_ON(dev->written); 958 wbi = dev->written = chosen; 959 spin_unlock(&sh->lock); 960 961 while (wbi && wbi->bi_sector < 962 dev->sector + STRIPE_SECTORS) { 963 tx = async_copy_data(1, wbi, dev->page, 964 dev->sector, tx); 965 wbi = r5_next_bio(wbi, dev->sector); 966 } 967 } 968 } 969 970 return tx; 971} 972 973static void ops_complete_reconstruct(void *stripe_head_ref) 974{ 975 struct stripe_head *sh = stripe_head_ref; 976 int disks = sh->disks; 977 int pd_idx = sh->pd_idx; 978 int qd_idx = sh->qd_idx; 979 int i; 980 981 pr_debug("%s: stripe %llu\n", __func__, 982 (unsigned long long)sh->sector); 983 984 for (i = disks; i--; ) { 985 struct r5dev *dev = &sh->dev[i]; 986 987 if (dev->written || i == pd_idx || i == qd_idx) 988 set_bit(R5_UPTODATE, &dev->flags); 989 } 990 991 if (sh->reconstruct_state == reconstruct_state_drain_run) 992 sh->reconstruct_state = reconstruct_state_drain_result; 993 else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) 994 sh->reconstruct_state = reconstruct_state_prexor_drain_result; 995 else { 996 BUG_ON(sh->reconstruct_state != reconstruct_state_run); 997 sh->reconstruct_state = reconstruct_state_result; 998 } 999 1000 set_bit(STRIPE_HANDLE, &sh->state); 1001 release_stripe(sh); 1002} 1003 1004static void 1005ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu, 1006 struct dma_async_tx_descriptor *tx) 1007{ 1008 int disks = sh->disks; 1009 struct page **xor_srcs = percpu->scribble; 1010 struct async_submit_ctl submit; 1011 int count = 0, pd_idx = sh->pd_idx, i; 1012 struct page *xor_dest; 1013 int prexor = 0; 1014 unsigned long flags; 1015 1016 pr_debug("%s: stripe %llu\n", __func__, 1017 (unsigned long long)sh->sector); 1018 1019 /* check if prexor is active which means only process blocks 1020 * that are part of a read-modify-write (written) 1021 */ 1022 if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) { 1023 prexor = 1; 1024 xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; 1025 for (i = disks; i--; ) { 1026 struct r5dev *dev = &sh->dev[i]; 1027 if (dev->written) 1028 xor_srcs[count++] = dev->page; 1029 } 1030 } else { 1031 xor_dest = sh->dev[pd_idx].page; 1032 for (i = disks; i--; ) { 1033 struct r5dev *dev = &sh->dev[i]; 1034 if (i != pd_idx) 1035 xor_srcs[count++] = dev->page; 1036 } 1037 } 1038 1039 /* 1/ if we prexor'd then the dest is reused as a source 1040 * 2/ if we did not prexor then we are redoing the parity 1041 * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST 1042 * for the synchronous xor case 1043 */ 1044 flags = ASYNC_TX_ACK | 1045 (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST); 1046 1047 atomic_inc(&sh->count); 1048 1049 init_async_submit(&submit, flags, tx, ops_complete_reconstruct, sh, 1050 to_addr_conv(sh, percpu)); 1051 if (unlikely(count == 1)) 1052 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); 1053 else 1054 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 1055} 1056 1057static void 1058ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu, 1059 struct dma_async_tx_descriptor *tx) 1060{ 1061 struct async_submit_ctl submit; 1062 struct page **blocks = percpu->scribble; 1063 int count; 1064 1065 pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); 1066 1067 count = set_syndrome_sources(blocks, sh); 1068 1069 atomic_inc(&sh->count); 1070 1071 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_reconstruct, 1072 sh, to_addr_conv(sh, percpu)); 1073 async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); 1074} 1075 1076static void ops_complete_check(void *stripe_head_ref) 1077{ 1078 struct stripe_head *sh = stripe_head_ref; 1079 1080 pr_debug("%s: stripe %llu\n", __func__, 1081 (unsigned long long)sh->sector); 1082 1083 sh->check_state = check_state_check_result; 1084 set_bit(STRIPE_HANDLE, &sh->state); 1085 release_stripe(sh); 1086} 1087 1088static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu) 1089{ 1090 int disks = sh->disks; 1091 int pd_idx = sh->pd_idx; 1092 int qd_idx = sh->qd_idx; 1093 struct page *xor_dest; 1094 struct page **xor_srcs = percpu->scribble; 1095 struct dma_async_tx_descriptor *tx; 1096 struct async_submit_ctl submit; 1097 int count; 1098 int i; 1099 1100 pr_debug("%s: stripe %llu\n", __func__, 1101 (unsigned long long)sh->sector); 1102 1103 count = 0; 1104 xor_dest = sh->dev[pd_idx].page; 1105 xor_srcs[count++] = xor_dest; 1106 for (i = disks; i--; ) { 1107 if (i == pd_idx || i == qd_idx) 1108 continue; 1109 xor_srcs[count++] = sh->dev[i].page; 1110 } 1111 1112 init_async_submit(&submit, 0, NULL, NULL, NULL, 1113 to_addr_conv(sh, percpu)); 1114 tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, 1115 &sh->ops.zero_sum_result, &submit); 1116 1117 atomic_inc(&sh->count); 1118 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL); 1119 tx = async_trigger_callback(&submit); 1120} 1121 1122static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp) 1123{ 1124 struct page **srcs = percpu->scribble; 1125 struct async_submit_ctl submit; 1126 int count; 1127 1128 pr_debug("%s: stripe %llu checkp: %d\n", __func__, 1129 (unsigned long long)sh->sector, checkp); 1130 1131 count = set_syndrome_sources(srcs, sh); 1132 if (!checkp) 1133 srcs[count] = NULL; 1134 1135 atomic_inc(&sh->count); 1136 init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check, 1137 sh, to_addr_conv(sh, percpu)); 1138 async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE, 1139 &sh->ops.zero_sum_result, percpu->spare_page, &submit); 1140} 1141 1142static void __raid_run_ops(struct stripe_head *sh, unsigned long ops_request) 1143{ 1144 int overlap_clear = 0, i, disks = sh->disks; 1145 struct dma_async_tx_descriptor *tx = NULL; 1146 raid5_conf_t *conf = sh->raid_conf; 1147 int level = conf->level; 1148 struct raid5_percpu *percpu; 1149 unsigned long cpu; 1150 1151 cpu = get_cpu(); 1152 percpu = per_cpu_ptr(conf->percpu, cpu); 1153 if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) { 1154 ops_run_biofill(sh); 1155 overlap_clear++; 1156 } 1157 1158 if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) { 1159 if (level < 6) 1160 tx = ops_run_compute5(sh, percpu); 1161 else { 1162 if (sh->ops.target2 < 0 || sh->ops.target < 0) 1163 tx = ops_run_compute6_1(sh, percpu); 1164 else 1165 tx = ops_run_compute6_2(sh, percpu); 1166 } 1167 /* terminate the chain if reconstruct is not set to be run */ 1168 if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) 1169 async_tx_ack(tx); 1170 } 1171 1172 if (test_bit(STRIPE_OP_PREXOR, &ops_request)) 1173 tx = ops_run_prexor(sh, percpu, tx); 1174 1175 if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) { 1176 tx = ops_run_biodrain(sh, tx); 1177 overlap_clear++; 1178 } 1179 1180 if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) { 1181 if (level < 6) 1182 ops_run_reconstruct5(sh, percpu, tx); 1183 else 1184 ops_run_reconstruct6(sh, percpu, tx); 1185 } 1186 1187 if (test_bit(STRIPE_OP_CHECK, &ops_request)) { 1188 if (sh->check_state == check_state_run) 1189 ops_run_check_p(sh, percpu); 1190 else if (sh->check_state == check_state_run_q) 1191 ops_run_check_pq(sh, percpu, 0); 1192 else if (sh->check_state == check_state_run_pq) 1193 ops_run_check_pq(sh, percpu, 1); 1194 else 1195 BUG(); 1196 } 1197 1198 if (overlap_clear) 1199 for (i = disks; i--; ) { 1200 struct r5dev *dev = &sh->dev[i]; 1201 if (test_and_clear_bit(R5_Overlap, &dev->flags)) 1202 wake_up(&sh->raid_conf->wait_for_overlap); 1203 } 1204 put_cpu(); 1205} 1206 1207#ifdef CONFIG_MULTICORE_RAID456 1208static void async_run_ops(void *param, async_cookie_t cookie) 1209{ 1210 struct stripe_head *sh = param; 1211 unsigned long ops_request = sh->ops.request; 1212 1213 clear_bit_unlock(STRIPE_OPS_REQ_PENDING, &sh->state); 1214 wake_up(&sh->ops.wait_for_ops); 1215 1216 __raid_run_ops(sh, ops_request); 1217 release_stripe(sh); 1218} 1219 1220static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request) 1221{ 1222 /* since handle_stripe can be called outside of raid5d context 1223 * we need to ensure sh->ops.request is de-staged before another 1224 * request arrives 1225 */ 1226 wait_event(sh->ops.wait_for_ops, 1227 !test_and_set_bit_lock(STRIPE_OPS_REQ_PENDING, &sh->state)); 1228 sh->ops.request = ops_request; 1229 1230 atomic_inc(&sh->count); 1231 async_schedule(async_run_ops, sh); 1232} 1233#else 1234#define raid_run_ops __raid_run_ops 1235#endif 1236 1237static int grow_one_stripe(raid5_conf_t *conf) 1238{ 1239 struct stripe_head *sh; 1240 sh = kmem_cache_alloc(conf->slab_cache, GFP_KERNEL); 1241 if (!sh) 1242 return 0; 1243 memset(sh, 0, sizeof(*sh) + (conf->raid_disks-1)*sizeof(struct r5dev)); 1244 sh->raid_conf = conf; 1245 spin_lock_init(&sh->lock); 1246 #ifdef CONFIG_MULTICORE_RAID456 1247 init_waitqueue_head(&sh->ops.wait_for_ops); 1248 #endif 1249 1250 if (grow_buffers(sh, conf->raid_disks)) { 1251 shrink_buffers(sh, conf->raid_disks); 1252 kmem_cache_free(conf->slab_cache, sh); 1253 return 0; 1254 } 1255 sh->disks = conf->raid_disks; 1256 /* we just created an active stripe so... */ 1257 atomic_set(&sh->count, 1); 1258 atomic_inc(&conf->active_stripes); 1259 INIT_LIST_HEAD(&sh->lru); 1260 release_stripe(sh); 1261 return 1; 1262} 1263 1264static int grow_stripes(raid5_conf_t *conf, int num) 1265{ 1266 struct kmem_cache *sc; 1267 int devs = conf->raid_disks; 1268 1269 sprintf(conf->cache_name[0], 1270 "raid%d-%s", conf->level, mdname(conf->mddev)); 1271 sprintf(conf->cache_name[1], 1272 "raid%d-%s-alt", conf->level, mdname(conf->mddev)); 1273 conf->active_name = 0; 1274 sc = kmem_cache_create(conf->cache_name[conf->active_name], 1275 sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev), 1276 0, 0, NULL); 1277 if (!sc) 1278 return 1; 1279 conf->slab_cache = sc; 1280 conf->pool_size = devs; 1281 while (num--) 1282 if (!grow_one_stripe(conf)) 1283 return 1; 1284 return 0; 1285} 1286 1287/** 1288 * scribble_len - return the required size of the scribble region 1289 * @num - total number of disks in the array 1290 * 1291 * The size must be enough to contain: 1292 * 1/ a struct page pointer for each device in the array +2 1293 * 2/ room to convert each entry in (1) to its corresponding dma 1294 * (dma_map_page()) or page (page_address()) address. 1295 * 1296 * Note: the +2 is for the destination buffers of the ddf/raid6 case where we 1297 * calculate over all devices (not just the data blocks), using zeros in place 1298 * of the P and Q blocks. 1299 */ 1300static size_t scribble_len(int num) 1301{ 1302 size_t len; 1303 1304 len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2); 1305 1306 return len; 1307} 1308 1309static int resize_stripes(raid5_conf_t *conf, int newsize) 1310{ 1311 /* Make all the stripes able to hold 'newsize' devices. 1312 * New slots in each stripe get 'page' set to a new page. 1313 * 1314 * This happens in stages: 1315 * 1/ create a new kmem_cache and allocate the required number of 1316 * stripe_heads. 1317 * 2/ gather all the old stripe_heads and tranfer the pages across 1318 * to the new stripe_heads. This will have the side effect of 1319 * freezing the array as once all stripe_heads have been collected, 1320 * no IO will be possible. Old stripe heads are freed once their 1321 * pages have been transferred over, and the old kmem_cache is 1322 * freed when all stripes are done. 1323 * 3/ reallocate conf->disks to be suitable bigger. If this fails, 1324 * we simple return a failre status - no need to clean anything up. 1325 * 4/ allocate new pages for the new slots in the new stripe_heads. 1326 * If this fails, we don't bother trying the shrink the 1327 * stripe_heads down again, we just leave them as they are. 1328 * As each stripe_head is processed the new one is released into 1329 * active service. 1330 * 1331 * Once step2 is started, we cannot afford to wait for a write, 1332 * so we use GFP_NOIO allocations. 1333 */ 1334 struct stripe_head *osh, *nsh; 1335 LIST_HEAD(newstripes); 1336 struct disk_info *ndisks; 1337 unsigned long cpu; 1338 int err; 1339 struct kmem_cache *sc; 1340 int i; 1341 1342 if (newsize <= conf->pool_size) 1343 return 0; /* never bother to shrink */ 1344 1345 err = md_allow_write(conf->mddev); 1346 if (err) 1347 return err; 1348 1349 /* Step 1 */ 1350 sc = kmem_cache_create(conf->cache_name[1-conf->active_name], 1351 sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev), 1352 0, 0, NULL); 1353 if (!sc) 1354 return -ENOMEM; 1355 1356 for (i = conf->max_nr_stripes; i; i--) { 1357 nsh = kmem_cache_alloc(sc, GFP_KERNEL); 1358 if (!nsh) 1359 break; 1360 1361 memset(nsh, 0, sizeof(*nsh) + (newsize-1)*sizeof(struct r5dev)); 1362 1363 nsh->raid_conf = conf; 1364 spin_lock_init(&nsh->lock); 1365 #ifdef CONFIG_MULTICORE_RAID456 1366 init_waitqueue_head(&nsh->ops.wait_for_ops); 1367 #endif 1368 1369 list_add(&nsh->lru, &newstripes); 1370 } 1371 if (i) { 1372 /* didn't get enough, give up */ 1373 while (!list_empty(&newstripes)) { 1374 nsh = list_entry(newstripes.next, struct stripe_head, lru); 1375 list_del(&nsh->lru); 1376 kmem_cache_free(sc, nsh); 1377 } 1378 kmem_cache_destroy(sc); 1379 return -ENOMEM; 1380 } 1381 /* Step 2 - Must use GFP_NOIO now. 1382 * OK, we have enough stripes, start collecting inactive 1383 * stripes and copying them over 1384 */ 1385 list_for_each_entry(nsh, &newstripes, lru) { 1386 spin_lock_irq(&conf->device_lock); 1387 wait_event_lock_irq(conf->wait_for_stripe, 1388 !list_empty(&conf->inactive_list), 1389 conf->device_lock, 1390 unplug_slaves(conf->mddev) 1391 ); 1392 osh = get_free_stripe(conf); 1393 spin_unlock_irq(&conf->device_lock); 1394 atomic_set(&nsh->count, 1); 1395 for(i=0; i<conf->pool_size; i++) 1396 nsh->dev[i].page = osh->dev[i].page; 1397 for( ; i<newsize; i++) 1398 nsh->dev[i].page = NULL; 1399 kmem_cache_free(conf->slab_cache, osh); 1400 } 1401 kmem_cache_destroy(conf->slab_cache); 1402 1403 /* Step 3. 1404 * At this point, we are holding all the stripes so the array 1405 * is completely stalled, so now is a good time to resize 1406 * conf->disks and the scribble region 1407 */ 1408 ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO); 1409 if (ndisks) { 1410 for (i=0; i<conf->raid_disks; i++) 1411 ndisks[i] = conf->disks[i]; 1412 kfree(conf->disks); 1413 conf->disks = ndisks; 1414 } else 1415 err = -ENOMEM; 1416 1417 get_online_cpus(); 1418 conf->scribble_len = scribble_len(newsize); 1419 for_each_present_cpu(cpu) { 1420 struct raid5_percpu *percpu; 1421 void *scribble; 1422 1423 percpu = per_cpu_ptr(conf->percpu, cpu); 1424 scribble = kmalloc(conf->scribble_len, GFP_NOIO); 1425 1426 if (scribble) { 1427 kfree(percpu->scribble); 1428 percpu->scribble = scribble; 1429 } else { 1430 err = -ENOMEM; 1431 break; 1432 } 1433 } 1434 put_online_cpus(); 1435 1436 /* Step 4, return new stripes to service */ 1437 while(!list_empty(&newstripes)) { 1438 nsh = list_entry(newstripes.next, struct stripe_head, lru); 1439 list_del_init(&nsh->lru); 1440 1441 for (i=conf->raid_disks; i < newsize; i++) 1442 if (nsh->dev[i].page == NULL) { 1443 struct page *p = alloc_page(GFP_NOIO); 1444 nsh->dev[i].page = p; 1445 if (!p) 1446 err = -ENOMEM; 1447 } 1448 release_stripe(nsh); 1449 } 1450 /* critical section pass, GFP_NOIO no longer needed */ 1451 1452 conf->slab_cache = sc; 1453 conf->active_name = 1-conf->active_name; 1454 conf->pool_size = newsize; 1455 return err; 1456} 1457 1458static int drop_one_stripe(raid5_conf_t *conf) 1459{ 1460 struct stripe_head *sh; 1461 1462 spin_lock_irq(&conf->device_lock); 1463 sh = get_free_stripe(conf); 1464 spin_unlock_irq(&conf->device_lock); 1465 if (!sh) 1466 return 0; 1467 BUG_ON(atomic_read(&sh->count)); 1468 shrink_buffers(sh, conf->pool_size); 1469 kmem_cache_free(conf->slab_cache, sh); 1470 atomic_dec(&conf->active_stripes); 1471 return 1; 1472} 1473 1474static void shrink_stripes(raid5_conf_t *conf) 1475{ 1476 while (drop_one_stripe(conf)) 1477 ; 1478 1479 if (conf->slab_cache) 1480 kmem_cache_destroy(conf->slab_cache); 1481 conf->slab_cache = NULL; 1482} 1483 1484static void raid5_end_read_request(struct bio * bi, int error) 1485{ 1486 struct stripe_head *sh = bi->bi_private; 1487 raid5_conf_t *conf = sh->raid_conf; 1488 int disks = sh->disks, i; 1489 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 1490 char b[BDEVNAME_SIZE]; 1491 mdk_rdev_t *rdev; 1492 1493 1494 for (i=0 ; i<disks; i++) 1495 if (bi == &sh->dev[i].req) 1496 break; 1497 1498 pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n", 1499 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 1500 uptodate); 1501 if (i == disks) { 1502 BUG(); 1503 return; 1504 } 1505 1506 if (uptodate) { 1507 set_bit(R5_UPTODATE, &sh->dev[i].flags); 1508 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 1509 rdev = conf->disks[i].rdev; 1510 printk_rl(KERN_INFO "raid5:%s: read error corrected" 1511 " (%lu sectors at %llu on %s)\n", 1512 mdname(conf->mddev), STRIPE_SECTORS, 1513 (unsigned long long)(sh->sector 1514 + rdev->data_offset), 1515 bdevname(rdev->bdev, b)); 1516 clear_bit(R5_ReadError, &sh->dev[i].flags); 1517 clear_bit(R5_ReWrite, &sh->dev[i].flags); 1518 } 1519 if (atomic_read(&conf->disks[i].rdev->read_errors)) 1520 atomic_set(&conf->disks[i].rdev->read_errors, 0); 1521 } else { 1522 const char *bdn = bdevname(conf->disks[i].rdev->bdev, b); 1523 int retry = 0; 1524 rdev = conf->disks[i].rdev; 1525 1526 clear_bit(R5_UPTODATE, &sh->dev[i].flags); 1527 atomic_inc(&rdev->read_errors); 1528 if (conf->mddev->degraded) 1529 printk_rl(KERN_WARNING 1530 "raid5:%s: read error not correctable " 1531 "(sector %llu on %s).\n", 1532 mdname(conf->mddev), 1533 (unsigned long long)(sh->sector 1534 + rdev->data_offset), 1535 bdn); 1536 else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) 1537 /* Oh, no!!! */ 1538 printk_rl(KERN_WARNING 1539 "raid5:%s: read error NOT corrected!! " 1540 "(sector %llu on %s).\n", 1541 mdname(conf->mddev), 1542 (unsigned long long)(sh->sector 1543 + rdev->data_offset), 1544 bdn); 1545 else if (atomic_read(&rdev->read_errors) 1546 > conf->max_nr_stripes) 1547 printk(KERN_WARNING 1548 "raid5:%s: Too many read errors, failing device %s.\n", 1549 mdname(conf->mddev), bdn); 1550 else 1551 retry = 1; 1552 if (retry) 1553 set_bit(R5_ReadError, &sh->dev[i].flags); 1554 else { 1555 clear_bit(R5_ReadError, &sh->dev[i].flags); 1556 clear_bit(R5_ReWrite, &sh->dev[i].flags); 1557 md_error(conf->mddev, rdev); 1558 } 1559 } 1560 rdev_dec_pending(conf->disks[i].rdev, conf->mddev); 1561 clear_bit(R5_LOCKED, &sh->dev[i].flags); 1562 set_bit(STRIPE_HANDLE, &sh->state); 1563 release_stripe(sh); 1564} 1565 1566static void raid5_end_write_request(struct bio *bi, int error) 1567{ 1568 struct stripe_head *sh = bi->bi_private; 1569 raid5_conf_t *conf = sh->raid_conf; 1570 int disks = sh->disks, i; 1571 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 1572 1573 for (i=0 ; i<disks; i++) 1574 if (bi == &sh->dev[i].req) 1575 break; 1576 1577 pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n", 1578 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 1579 uptodate); 1580 if (i == disks) { 1581 BUG(); 1582 return; 1583 } 1584 1585 if (!uptodate) 1586 md_error(conf->mddev, conf->disks[i].rdev); 1587 1588 rdev_dec_pending(conf->disks[i].rdev, conf->mddev); 1589 1590 clear_bit(R5_LOCKED, &sh->dev[i].flags); 1591 set_bit(STRIPE_HANDLE, &sh->state); 1592 release_stripe(sh); 1593} 1594 1595 1596static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous); 1597 1598static void raid5_build_block(struct stripe_head *sh, int i, int previous) 1599{ 1600 struct r5dev *dev = &sh->dev[i]; 1601 1602 bio_init(&dev->req); 1603 dev->req.bi_io_vec = &dev->vec; 1604 dev->req.bi_vcnt++; 1605 dev->req.bi_max_vecs++; 1606 dev->vec.bv_page = dev->page; 1607 dev->vec.bv_len = STRIPE_SIZE; 1608 dev->vec.bv_offset = 0; 1609 1610 dev->req.bi_sector = sh->sector; 1611 dev->req.bi_private = sh; 1612 1613 dev->flags = 0; 1614 dev->sector = compute_blocknr(sh, i, previous); 1615} 1616 1617static void error(mddev_t *mddev, mdk_rdev_t *rdev) 1618{ 1619 char b[BDEVNAME_SIZE]; 1620 raid5_conf_t *conf = (raid5_conf_t *) mddev->private; 1621 pr_debug("raid5: error called\n"); 1622 1623 if (!test_bit(Faulty, &rdev->flags)) { 1624 set_bit(MD_CHANGE_DEVS, &mddev->flags); 1625 if (test_and_clear_bit(In_sync, &rdev->flags)) { 1626 unsigned long flags; 1627 spin_lock_irqsave(&conf->device_lock, flags); 1628 mddev->degraded++; 1629 spin_unlock_irqrestore(&conf->device_lock, flags); 1630 /* 1631 * if recovery was running, make sure it aborts. 1632 */ 1633 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 1634 } 1635 set_bit(Faulty, &rdev->flags); 1636 printk(KERN_ALERT 1637 "raid5: Disk failure on %s, disabling device.\n" 1638 "raid5: Operation continuing on %d devices.\n", 1639 bdevname(rdev->bdev,b), conf->raid_disks - mddev->degraded); 1640 } 1641} 1642 1643/* 1644 * Input: a 'big' sector number, 1645 * Output: index of the data and parity disk, and the sector # in them. 1646 */ 1647static sector_t raid5_compute_sector(raid5_conf_t *conf, sector_t r_sector, 1648 int previous, int *dd_idx, 1649 struct stripe_head *sh) 1650{ 1651 long stripe; 1652 unsigned long chunk_number; 1653 unsigned int chunk_offset; 1654 int pd_idx, qd_idx; 1655 int ddf_layout = 0; 1656 sector_t new_sector; 1657 int algorithm = previous ? conf->prev_algo 1658 : conf->algorithm; 1659 int sectors_per_chunk = previous ? conf->prev_chunk_sectors 1660 : conf->chunk_sectors; 1661 int raid_disks = previous ? conf->previous_raid_disks 1662 : conf->raid_disks; 1663 int data_disks = raid_disks - conf->max_degraded; 1664 1665 /* First compute the information on this sector */ 1666 1667 /* 1668 * Compute the chunk number and the sector offset inside the chunk 1669 */ 1670 chunk_offset = sector_div(r_sector, sectors_per_chunk); 1671 chunk_number = r_sector; 1672 BUG_ON(r_sector != chunk_number); 1673 1674 /* 1675 * Compute the stripe number 1676 */ 1677 stripe = chunk_number / data_disks; 1678 1679 /* 1680 * Compute the data disk and parity disk indexes inside the stripe 1681 */ 1682 *dd_idx = chunk_number % data_disks; 1683 1684 /* 1685 * Select the parity disk based on the user selected algorithm. 1686 */ 1687 pd_idx = qd_idx = ~0; 1688 switch(conf->level) { 1689 case 4: 1690 pd_idx = data_disks; 1691 break; 1692 case 5: 1693 switch (algorithm) { 1694 case ALGORITHM_LEFT_ASYMMETRIC: 1695 pd_idx = data_disks - stripe % raid_disks; 1696 if (*dd_idx >= pd_idx) 1697 (*dd_idx)++; 1698 break; 1699 case ALGORITHM_RIGHT_ASYMMETRIC: 1700 pd_idx = stripe % raid_disks; 1701 if (*dd_idx >= pd_idx) 1702 (*dd_idx)++; 1703 break; 1704 case ALGORITHM_LEFT_SYMMETRIC: 1705 pd_idx = data_disks - stripe % raid_disks; 1706 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 1707 break; 1708 case ALGORITHM_RIGHT_SYMMETRIC: 1709 pd_idx = stripe % raid_disks; 1710 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 1711 break; 1712 case ALGORITHM_PARITY_0: 1713 pd_idx = 0; 1714 (*dd_idx)++; 1715 break; 1716 case ALGORITHM_PARITY_N: 1717 pd_idx = data_disks; 1718 break; 1719 default: 1720 printk(KERN_ERR "raid5: unsupported algorithm %d\n", 1721 algorithm); 1722 BUG(); 1723 } 1724 break; 1725 case 6: 1726 1727 switch (algorithm) { 1728 case ALGORITHM_LEFT_ASYMMETRIC: 1729 pd_idx = raid_disks - 1 - (stripe % raid_disks); 1730 qd_idx = pd_idx + 1; 1731 if (pd_idx == raid_disks-1) { 1732 (*dd_idx)++; /* Q D D D P */ 1733 qd_idx = 0; 1734 } else if (*dd_idx >= pd_idx) 1735 (*dd_idx) += 2; /* D D P Q D */ 1736 break; 1737 case ALGORITHM_RIGHT_ASYMMETRIC: 1738 pd_idx = stripe % raid_disks; 1739 qd_idx = pd_idx + 1; 1740 if (pd_idx == raid_disks-1) { 1741 (*dd_idx)++; /* Q D D D P */ 1742 qd_idx = 0; 1743 } else if (*dd_idx >= pd_idx) 1744 (*dd_idx) += 2; /* D D P Q D */ 1745 break; 1746 case ALGORITHM_LEFT_SYMMETRIC: 1747 pd_idx = raid_disks - 1 - (stripe % raid_disks); 1748 qd_idx = (pd_idx + 1) % raid_disks; 1749 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; 1750 break; 1751 case ALGORITHM_RIGHT_SYMMETRIC: 1752 pd_idx = stripe % raid_disks; 1753 qd_idx = (pd_idx + 1) % raid_disks; 1754 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; 1755 break; 1756 1757 case ALGORITHM_PARITY_0: 1758 pd_idx = 0; 1759 qd_idx = 1; 1760 (*dd_idx) += 2; 1761 break; 1762 case ALGORITHM_PARITY_N: 1763 pd_idx = data_disks; 1764 qd_idx = data_disks + 1; 1765 break; 1766 1767 case ALGORITHM_ROTATING_ZERO_RESTART: 1768 /* Exactly the same as RIGHT_ASYMMETRIC, but or 1769 * of blocks for computing Q is different. 1770 */ 1771 pd_idx = stripe % raid_disks; 1772 qd_idx = pd_idx + 1; 1773 if (pd_idx == raid_disks-1) { 1774 (*dd_idx)++; /* Q D D D P */ 1775 qd_idx = 0; 1776 } else if (*dd_idx >= pd_idx) 1777 (*dd_idx) += 2; /* D D P Q D */ 1778 ddf_layout = 1; 1779 break; 1780 1781 case ALGORITHM_ROTATING_N_RESTART: 1782 /* Same a left_asymmetric, by first stripe is 1783 * D D D P Q rather than 1784 * Q D D D P 1785 */ 1786 pd_idx = raid_disks - 1 - ((stripe + 1) % raid_disks); 1787 qd_idx = pd_idx + 1; 1788 if (pd_idx == raid_disks-1) { 1789 (*dd_idx)++; /* Q D D D P */ 1790 qd_idx = 0; 1791 } else if (*dd_idx >= pd_idx) 1792 (*dd_idx) += 2; /* D D P Q D */ 1793 ddf_layout = 1; 1794 break; 1795 1796 case ALGORITHM_ROTATING_N_CONTINUE: 1797 /* Same as left_symmetric but Q is before P */ 1798 pd_idx = raid_disks - 1 - (stripe % raid_disks); 1799 qd_idx = (pd_idx + raid_disks - 1) % raid_disks; 1800 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 1801 ddf_layout = 1; 1802 break; 1803 1804 case ALGORITHM_LEFT_ASYMMETRIC_6: 1805 /* RAID5 left_asymmetric, with Q on last device */ 1806 pd_idx = data_disks - stripe % (raid_disks-1); 1807 if (*dd_idx >= pd_idx) 1808 (*dd_idx)++; 1809 qd_idx = raid_disks - 1; 1810 break; 1811 1812 case ALGORITHM_RIGHT_ASYMMETRIC_6: 1813 pd_idx = stripe % (raid_disks-1); 1814 if (*dd_idx >= pd_idx) 1815 (*dd_idx)++; 1816 qd_idx = raid_disks - 1; 1817 break; 1818 1819 case ALGORITHM_LEFT_SYMMETRIC_6: 1820 pd_idx = data_disks - stripe % (raid_disks-1); 1821 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); 1822 qd_idx = raid_disks - 1; 1823 break; 1824 1825 case ALGORITHM_RIGHT_SYMMETRIC_6: 1826 pd_idx = stripe % (raid_disks-1); 1827 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); 1828 qd_idx = raid_disks - 1; 1829 break; 1830 1831 case ALGORITHM_PARITY_0_6: 1832 pd_idx = 0; 1833 (*dd_idx)++; 1834 qd_idx = raid_disks - 1; 1835 break; 1836 1837 1838 default: 1839 printk(KERN_CRIT "raid6: unsupported algorithm %d\n", 1840 algorithm); 1841 BUG(); 1842 } 1843 break; 1844 } 1845 1846 if (sh) { 1847 sh->pd_idx = pd_idx; 1848 sh->qd_idx = qd_idx; 1849 sh->ddf_layout = ddf_layout; 1850 } 1851 /* 1852 * Finally, compute the new sector number 1853 */ 1854 new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset; 1855 return new_sector; 1856} 1857 1858 1859static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous) 1860{ 1861 raid5_conf_t *conf = sh->raid_conf; 1862 int raid_disks = sh->disks; 1863 int data_disks = raid_disks - conf->max_degraded; 1864 sector_t new_sector = sh->sector, check; 1865 int sectors_per_chunk = previous ? conf->prev_chunk_sectors 1866 : conf->chunk_sectors; 1867 int algorithm = previous ? conf->prev_algo 1868 : conf->algorithm; 1869 sector_t stripe; 1870 int chunk_offset; 1871 int chunk_number, dummy1, dd_idx = i; 1872 sector_t r_sector; 1873 struct stripe_head sh2; 1874 1875 1876 chunk_offset = sector_div(new_sector, sectors_per_chunk); 1877 stripe = new_sector; 1878 BUG_ON(new_sector != stripe); 1879 1880 if (i == sh->pd_idx) 1881 return 0; 1882 switch(conf->level) { 1883 case 4: break; 1884 case 5: 1885 switch (algorithm) { 1886 case ALGORITHM_LEFT_ASYMMETRIC: 1887 case ALGORITHM_RIGHT_ASYMMETRIC: 1888 if (i > sh->pd_idx) 1889 i--; 1890 break; 1891 case ALGORITHM_LEFT_SYMMETRIC: 1892 case ALGORITHM_RIGHT_SYMMETRIC: 1893 if (i < sh->pd_idx) 1894 i += raid_disks; 1895 i -= (sh->pd_idx + 1); 1896 break; 1897 case ALGORITHM_PARITY_0: 1898 i -= 1; 1899 break; 1900 case ALGORITHM_PARITY_N: 1901 break; 1902 default: 1903 printk(KERN_ERR "raid5: unsupported algorithm %d\n", 1904 algorithm); 1905 BUG(); 1906 } 1907 break; 1908 case 6: 1909 if (i == sh->qd_idx) 1910 return 0; /* It is the Q disk */ 1911 switch (algorithm) { 1912 case ALGORITHM_LEFT_ASYMMETRIC: 1913 case ALGORITHM_RIGHT_ASYMMETRIC: 1914 case ALGORITHM_ROTATING_ZERO_RESTART: 1915 case ALGORITHM_ROTATING_N_RESTART: 1916 if (sh->pd_idx == raid_disks-1) 1917 i--; /* Q D D D P */ 1918 else if (i > sh->pd_idx) 1919 i -= 2; /* D D P Q D */ 1920 break; 1921 case ALGORITHM_LEFT_SYMMETRIC: 1922 case ALGORITHM_RIGHT_SYMMETRIC: 1923 if (sh->pd_idx == raid_disks-1) 1924 i--; /* Q D D D P */ 1925 else { 1926 /* D D P Q D */ 1927 if (i < sh->pd_idx) 1928 i += raid_disks; 1929 i -= (sh->pd_idx + 2); 1930 } 1931 break; 1932 case ALGORITHM_PARITY_0: 1933 i -= 2; 1934 break; 1935 case ALGORITHM_PARITY_N: 1936 break; 1937 case ALGORITHM_ROTATING_N_CONTINUE: 1938 if (sh->pd_idx == 0) 1939 i--; /* P D D D Q */ 1940 else if (i > sh->pd_idx) 1941 i -= 2; /* D D Q P D */ 1942 break; 1943 case ALGORITHM_LEFT_ASYMMETRIC_6: 1944 case ALGORITHM_RIGHT_ASYMMETRIC_6: 1945 if (i > sh->pd_idx) 1946 i--; 1947 break; 1948 case ALGORITHM_LEFT_SYMMETRIC_6: 1949 case ALGORITHM_RIGHT_SYMMETRIC_6: 1950 if (i < sh->pd_idx) 1951 i += data_disks + 1; 1952 i -= (sh->pd_idx + 1); 1953 break; 1954 case ALGORITHM_PARITY_0_6: 1955 i -= 1; 1956 break; 1957 default: 1958 printk(KERN_CRIT "raid6: unsupported algorithm %d\n", 1959 algorithm); 1960 BUG(); 1961 } 1962 break; 1963 } 1964 1965 chunk_number = stripe * data_disks + i; 1966 r_sector = (sector_t)chunk_number * sectors_per_chunk + chunk_offset; 1967 1968 check = raid5_compute_sector(conf, r_sector, 1969 previous, &dummy1, &sh2); 1970 if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx 1971 || sh2.qd_idx != sh->qd_idx) { 1972 printk(KERN_ERR "compute_blocknr: map not correct\n"); 1973 return 0; 1974 } 1975 return r_sector; 1976} 1977 1978 1979static void 1980schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s, 1981 int rcw, int expand) 1982{ 1983 int i, pd_idx = sh->pd_idx, disks = sh->disks; 1984 raid5_conf_t *conf = sh->raid_conf; 1985 int level = conf->level; 1986 1987 if (rcw) { 1988 /* if we are not expanding this is a proper write request, and 1989 * there will be bios with new data to be drained into the 1990 * stripe cache 1991 */ 1992 if (!expand) { 1993 sh->reconstruct_state = reconstruct_state_drain_run; 1994 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); 1995 } else 1996 sh->reconstruct_state = reconstruct_state_run; 1997 1998 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); 1999 2000 for (i = disks; i--; ) { 2001 struct r5dev *dev = &sh->dev[i]; 2002 2003 if (dev->towrite) { 2004 set_bit(R5_LOCKED, &dev->flags); 2005 set_bit(R5_Wantdrain, &dev->flags); 2006 if (!expand) 2007 clear_bit(R5_UPTODATE, &dev->flags); 2008 s->locked++; 2009 } 2010 } 2011 if (s->locked + conf->max_degraded == disks) 2012 if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state)) 2013 atomic_inc(&conf->pending_full_writes); 2014 } else { 2015 BUG_ON(level == 6); 2016 BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) || 2017 test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags))); 2018 2019 sh->reconstruct_state = reconstruct_state_prexor_drain_run; 2020 set_bit(STRIPE_OP_PREXOR, &s->ops_request); 2021 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); 2022 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); 2023 2024 for (i = disks; i--; ) { 2025 struct r5dev *dev = &sh->dev[i]; 2026 if (i == pd_idx) 2027 continue; 2028 2029 if (dev->towrite && 2030 (test_bit(R5_UPTODATE, &dev->flags) || 2031 test_bit(R5_Wantcompute, &dev->flags))) { 2032 set_bit(R5_Wantdrain, &dev->flags); 2033 set_bit(R5_LOCKED, &dev->flags); 2034 clear_bit(R5_UPTODATE, &dev->flags); 2035 s->locked++; 2036 } 2037 } 2038 } 2039 2040 /* keep the parity disk(s) locked while asynchronous operations 2041 * are in flight 2042 */ 2043 set_bit(R5_LOCKED, &sh->dev[pd_idx].flags); 2044 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 2045 s->locked++; 2046 2047 if (level == 6) { 2048 int qd_idx = sh->qd_idx; 2049 struct r5dev *dev = &sh->dev[qd_idx]; 2050 2051 set_bit(R5_LOCKED, &dev->flags); 2052 clear_bit(R5_UPTODATE, &dev->flags); 2053 s->locked++; 2054 } 2055 2056 pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n", 2057 __func__, (unsigned long long)sh->sector, 2058 s->locked, s->ops_request); 2059} 2060 2061/* 2062 * Each stripe/dev can have one or more bion attached. 2063 * toread/towrite point to the first in a chain. 2064 * The bi_next chain must be in order. 2065 */ 2066static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, int forwrite) 2067{ 2068 struct bio **bip; 2069 raid5_conf_t *conf = sh->raid_conf; 2070 int firstwrite=0; 2071 2072 pr_debug("adding bh b#%llu to stripe s#%llu\n", 2073 (unsigned long long)bi->bi_sector, 2074 (unsigned long long)sh->sector); 2075 2076 2077 spin_lock(&sh->lock); 2078 spin_lock_irq(&conf->device_lock); 2079 if (forwrite) { 2080 bip = &sh->dev[dd_idx].towrite; 2081 if (*bip == NULL && sh->dev[dd_idx].written == NULL) 2082 firstwrite = 1; 2083 } else 2084 bip = &sh->dev[dd_idx].toread; 2085 while (*bip && (*bip)->bi_sector < bi->bi_sector) { 2086 if ((*bip)->bi_sector + ((*bip)->bi_size >> 9) > bi->bi_sector) 2087 goto overlap; 2088 bip = & (*bip)->bi_next; 2089 } 2090 if (*bip && (*bip)->bi_sector < bi->bi_sector + ((bi->bi_size)>>9)) 2091 goto overlap; 2092 2093 BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next); 2094 if (*bip) 2095 bi->bi_next = *bip; 2096 *bip = bi; 2097 bi->bi_phys_segments++; 2098 spin_unlock_irq(&conf->device_lock); 2099 spin_unlock(&sh->lock); 2100 2101 pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n", 2102 (unsigned long long)bi->bi_sector, 2103 (unsigned long long)sh->sector, dd_idx); 2104 2105 if (conf->mddev->bitmap && firstwrite) { 2106 bitmap_startwrite(conf->mddev->bitmap, sh->sector, 2107 STRIPE_SECTORS, 0); 2108 sh->bm_seq = conf->seq_flush+1; 2109 set_bit(STRIPE_BIT_DELAY, &sh->state); 2110 } 2111 2112 if (forwrite) { 2113 /* check if page is covered */ 2114 sector_t sector = sh->dev[dd_idx].sector; 2115 for (bi=sh->dev[dd_idx].towrite; 2116 sector < sh->dev[dd_idx].sector + STRIPE_SECTORS && 2117 bi && bi->bi_sector <= sector; 2118 bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) { 2119 if (bi->bi_sector + (bi->bi_size>>9) >= sector) 2120 sector = bi->bi_sector + (bi->bi_size>>9); 2121 } 2122 if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS) 2123 set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags); 2124 } 2125 return 1; 2126 2127 overlap: 2128 set_bit(R5_Overlap, &sh->dev[dd_idx].flags); 2129 spin_unlock_irq(&conf->device_lock); 2130 spin_unlock(&sh->lock); 2131 return 0; 2132} 2133 2134static void end_reshape(raid5_conf_t *conf); 2135 2136static void stripe_set_idx(sector_t stripe, raid5_conf_t *conf, int previous, 2137 struct stripe_head *sh) 2138{ 2139 int sectors_per_chunk = 2140 previous ? conf->prev_chunk_sectors : conf->chunk_sectors; 2141 int dd_idx; 2142 int chunk_offset = sector_div(stripe, sectors_per_chunk); 2143 int disks = previous ? conf->previous_raid_disks : conf->raid_disks; 2144 2145 raid5_compute_sector(conf, 2146 stripe * (disks - conf->max_degraded) 2147 *sectors_per_chunk + chunk_offset, 2148 previous, 2149 &dd_idx, sh); 2150} 2151 2152static void 2153handle_failed_stripe(raid5_conf_t *conf, struct stripe_head *sh, 2154 struct stripe_head_state *s, int disks, 2155 struct bio **return_bi) 2156{ 2157 int i; 2158 for (i = disks; i--; ) { 2159 struct bio *bi; 2160 int bitmap_end = 0; 2161 2162 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 2163 mdk_rdev_t *rdev; 2164 rcu_read_lock(); 2165 rdev = rcu_dereference(conf->disks[i].rdev); 2166 if (rdev && test_bit(In_sync, &rdev->flags)) 2167 /* multiple read failures in one stripe */ 2168 md_error(conf->mddev, rdev); 2169 rcu_read_unlock(); 2170 } 2171 spin_lock_irq(&conf->device_lock); 2172 /* fail all writes first */ 2173 bi = sh->dev[i].towrite; 2174 sh->dev[i].towrite = NULL; 2175 if (bi) { 2176 s->to_write--; 2177 bitmap_end = 1; 2178 } 2179 2180 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 2181 wake_up(&conf->wait_for_overlap); 2182 2183 while (bi && bi->bi_sector < 2184 sh->dev[i].sector + STRIPE_SECTORS) { 2185 struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector); 2186 clear_bit(BIO_UPTODATE, &bi->bi_flags); 2187 if (!raid5_dec_bi_phys_segments(bi)) { 2188 md_write_end(conf->mddev); 2189 bi->bi_next = *return_bi; 2190 *return_bi = bi; 2191 } 2192 bi = nextbi; 2193 } 2194 /* and fail all 'written' */ 2195 bi = sh->dev[i].written; 2196 sh->dev[i].written = NULL; 2197 if (bi) bitmap_end = 1; 2198 while (bi && bi->bi_sector < 2199 sh->dev[i].sector + STRIPE_SECTORS) { 2200 struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector); 2201 clear_bit(BIO_UPTODATE, &bi->bi_flags); 2202 if (!raid5_dec_bi_phys_segments(bi)) { 2203 md_write_end(conf->mddev); 2204 bi->bi_next = *return_bi; 2205 *return_bi = bi; 2206 } 2207 bi = bi2; 2208 } 2209 2210 /* fail any reads if this device is non-operational and 2211 * the data has not reached the cache yet. 2212 */ 2213 if (!test_bit(R5_Wantfill, &sh->dev[i].flags) && 2214 (!test_bit(R5_Insync, &sh->dev[i].flags) || 2215 test_bit(R5_ReadError, &sh->dev[i].flags))) { 2216 bi = sh->dev[i].toread; 2217 sh->dev[i].toread = NULL; 2218 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 2219 wake_up(&conf->wait_for_overlap); 2220 if (bi) s->to_read--; 2221 while (bi && bi->bi_sector < 2222 sh->dev[i].sector + STRIPE_SECTORS) { 2223 struct bio *nextbi = 2224 r5_next_bio(bi, sh->dev[i].sector); 2225 clear_bit(BIO_UPTODATE, &bi->bi_flags); 2226 if (!raid5_dec_bi_phys_segments(bi)) { 2227 bi->bi_next = *return_bi; 2228 *return_bi = bi; 2229 } 2230 bi = nextbi; 2231 } 2232 } 2233 spin_unlock_irq(&conf->device_lock); 2234 if (bitmap_end) 2235 bitmap_endwrite(conf->mddev->bitmap, sh->sector, 2236 STRIPE_SECTORS, 0, 0); 2237 } 2238 2239 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) 2240 if (atomic_dec_and_test(&conf->pending_full_writes)) 2241 md_wakeup_thread(conf->mddev->thread); 2242} 2243 2244/* fetch_block5 - checks the given member device to see if its data needs 2245 * to be read or computed to satisfy a request. 2246 * 2247 * Returns 1 when no more member devices need to be checked, otherwise returns 2248 * 0 to tell the loop in handle_stripe_fill5 to continue 2249 */ 2250static int fetch_block5(struct stripe_head *sh, struct stripe_head_state *s, 2251 int disk_idx, int disks) 2252{ 2253 struct r5dev *dev = &sh->dev[disk_idx]; 2254 struct r5dev *failed_dev = &sh->dev[s->failed_num]; 2255 2256 /* is the data in this block needed, and can we get it? */ 2257 if (!test_bit(R5_LOCKED, &dev->flags) && 2258 !test_bit(R5_UPTODATE, &dev->flags) && 2259 (dev->toread || 2260 (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) || 2261 s->syncing || s->expanding || 2262 (s->failed && 2263 (failed_dev->toread || 2264 (failed_dev->towrite && 2265 !test_bit(R5_OVERWRITE, &failed_dev->flags)))))) { 2266 /* We would like to get this block, possibly by computing it, 2267 * otherwise read it if the backing disk is insync 2268 */ 2269 if ((s->uptodate == disks - 1) && 2270 (s->failed && disk_idx == s->failed_num)) { 2271 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2272 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2273 set_bit(R5_Wantcompute, &dev->flags); 2274 sh->ops.target = disk_idx; 2275 sh->ops.target2 = -1; 2276 s->req_compute = 1; 2277 /* Careful: from this point on 'uptodate' is in the eye 2278 * of raid_run_ops which services 'compute' operations 2279 * before writes. R5_Wantcompute flags a block that will 2280 * be R5_UPTODATE by the time it is needed for a 2281 * subsequent operation. 2282 */ 2283 s->uptodate++; 2284 return 1; /* uptodate + compute == disks */ 2285 } else if (test_bit(R5_Insync, &dev->flags)) { 2286 set_bit(R5_LOCKED, &dev->flags); 2287 set_bit(R5_Wantread, &dev->flags); 2288 s->locked++; 2289 pr_debug("Reading block %d (sync=%d)\n", disk_idx, 2290 s->syncing); 2291 } 2292 } 2293 2294 return 0; 2295} 2296 2297/** 2298 * handle_stripe_fill5 - read or compute data to satisfy pending requests. 2299 */ 2300static void handle_stripe_fill5(struct stripe_head *sh, 2301 struct stripe_head_state *s, int disks) 2302{ 2303 int i; 2304 2305 /* look for blocks to read/compute, skip this if a compute 2306 * is already in flight, or if the stripe contents are in the 2307 * midst of changing due to a write 2308 */ 2309 if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state && 2310 !sh->reconstruct_state) 2311 for (i = disks; i--; ) 2312 if (fetch_block5(sh, s, i, disks)) 2313 break; 2314 set_bit(STRIPE_HANDLE, &sh->state); 2315} 2316 2317/* fetch_block6 - checks the given member device to see if its data needs 2318 * to be read or computed to satisfy a request. 2319 * 2320 * Returns 1 when no more member devices need to be checked, otherwise returns 2321 * 0 to tell the loop in handle_stripe_fill6 to continue 2322 */ 2323static int fetch_block6(struct stripe_head *sh, struct stripe_head_state *s, 2324 struct r6_state *r6s, int disk_idx, int disks) 2325{ 2326 struct r5dev *dev = &sh->dev[disk_idx]; 2327 struct r5dev *fdev[2] = { &sh->dev[r6s->failed_num[0]], 2328 &sh->dev[r6s->failed_num[1]] }; 2329 2330 if (!test_bit(R5_LOCKED, &dev->flags) && 2331 !test_bit(R5_UPTODATE, &dev->flags) && 2332 (dev->toread || 2333 (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) || 2334 s->syncing || s->expanding || 2335 (s->failed >= 1 && 2336 (fdev[0]->toread || s->to_write)) || 2337 (s->failed >= 2 && 2338 (fdev[1]->toread || s->to_write)))) { 2339 /* we would like to get this block, possibly by computing it, 2340 * otherwise read it if the backing disk is insync 2341 */ 2342 BUG_ON(test_bit(R5_Wantcompute, &dev->flags)); 2343 BUG_ON(test_bit(R5_Wantread, &dev->flags)); 2344 if ((s->uptodate == disks - 1) && 2345 (s->failed && (disk_idx == r6s->failed_num[0] || 2346 disk_idx == r6s->failed_num[1]))) { 2347 /* have disk failed, and we're requested to fetch it; 2348 * do compute it 2349 */ 2350 pr_debug("Computing stripe %llu block %d\n", 2351 (unsigned long long)sh->sector, disk_idx); 2352 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2353 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2354 set_bit(R5_Wantcompute, &dev->flags); 2355 sh->ops.target = disk_idx; 2356 sh->ops.target2 = -1; /* no 2nd target */ 2357 s->req_compute = 1; 2358 s->uptodate++; 2359 return 1; 2360 } else if (s->uptodate == disks-2 && s->failed >= 2) { 2361 /* Computing 2-failure is *very* expensive; only 2362 * do it if failed >= 2 2363 */ 2364 int other; 2365 for (other = disks; other--; ) { 2366 if (other == disk_idx) 2367 continue; 2368 if (!test_bit(R5_UPTODATE, 2369 &sh->dev[other].flags)) 2370 break; 2371 } 2372 BUG_ON(other < 0); 2373 pr_debug("Computing stripe %llu blocks %d,%d\n", 2374 (unsigned long long)sh->sector, 2375 disk_idx, other); 2376 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2377 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2378 set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags); 2379 set_bit(R5_Wantcompute, &sh->dev[other].flags); 2380 sh->ops.target = disk_idx; 2381 sh->ops.target2 = other; 2382 s->uptodate += 2; 2383 s->req_compute = 1; 2384 return 1; 2385 } else if (test_bit(R5_Insync, &dev->flags)) { 2386 set_bit(R5_LOCKED, &dev->flags); 2387 set_bit(R5_Wantread, &dev->flags); 2388 s->locked++; 2389 pr_debug("Reading block %d (sync=%d)\n", 2390 disk_idx, s->syncing); 2391 } 2392 } 2393 2394 return 0; 2395} 2396 2397/** 2398 * handle_stripe_fill6 - read or compute data to satisfy pending requests. 2399 */ 2400static void handle_stripe_fill6(struct stripe_head *sh, 2401 struct stripe_head_state *s, struct r6_state *r6s, 2402 int disks) 2403{ 2404 int i; 2405 2406 /* look for blocks to read/compute, skip this if a compute 2407 * is already in flight, or if the stripe contents are in the 2408 * midst of changing due to a write 2409 */ 2410 if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state && 2411 !sh->reconstruct_state) 2412 for (i = disks; i--; ) 2413 if (fetch_block6(sh, s, r6s, i, disks)) 2414 break; 2415 set_bit(STRIPE_HANDLE, &sh->state); 2416} 2417 2418 2419/* handle_stripe_clean_event 2420 * any written block on an uptodate or failed drive can be returned. 2421 * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but 2422 * never LOCKED, so we don't need to test 'failed' directly. 2423 */ 2424static void handle_stripe_clean_event(raid5_conf_t *conf, 2425 struct stripe_head *sh, int disks, struct bio **return_bi) 2426{ 2427 int i; 2428 struct r5dev *dev; 2429 2430 for (i = disks; i--; ) 2431 if (sh->dev[i].written) { 2432 dev = &sh->dev[i]; 2433 if (!test_bit(R5_LOCKED, &dev->flags) && 2434 test_bit(R5_UPTODATE, &dev->flags)) { 2435 /* We can return any write requests */ 2436 struct bio *wbi, *wbi2; 2437 int bitmap_end = 0; 2438 pr_debug("Return write for disc %d\n", i); 2439 spin_lock_irq(&conf->device_lock); 2440 wbi = dev->written; 2441 dev->written = NULL; 2442 while (wbi && wbi->bi_sector < 2443 dev->sector + STRIPE_SECTORS) { 2444 wbi2 = r5_next_bio(wbi, dev->sector); 2445 if (!raid5_dec_bi_phys_segments(wbi)) { 2446 md_write_end(conf->mddev); 2447 wbi->bi_next = *return_bi; 2448 *return_bi = wbi; 2449 } 2450 wbi = wbi2; 2451 } 2452 if (dev->towrite == NULL) 2453 bitmap_end = 1; 2454 spin_unlock_irq(&conf->device_lock); 2455 if (bitmap_end) 2456 bitmap_endwrite(conf->mddev->bitmap, 2457 sh->sector, 2458 STRIPE_SECTORS, 2459 !test_bit(STRIPE_DEGRADED, &sh->state), 2460 0); 2461 } 2462 } 2463 2464 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) 2465 if (atomic_dec_and_test(&conf->pending_full_writes)) 2466 md_wakeup_thread(conf->mddev->thread); 2467} 2468 2469static void handle_stripe_dirtying5(raid5_conf_t *conf, 2470 struct stripe_head *sh, struct stripe_head_state *s, int disks) 2471{ 2472 int rmw = 0, rcw = 0, i; 2473 for (i = disks; i--; ) { 2474 /* would I have to read this buffer for read_modify_write */ 2475 struct r5dev *dev = &sh->dev[i]; 2476 if ((dev->towrite || i == sh->pd_idx) && 2477 !test_bit(R5_LOCKED, &dev->flags) && 2478 !(test_bit(R5_UPTODATE, &dev->flags) || 2479 test_bit(R5_Wantcompute, &dev->flags))) { 2480 if (test_bit(R5_Insync, &dev->flags)) 2481 rmw++; 2482 else 2483 rmw += 2*disks; /* cannot read it */ 2484 } 2485 /* Would I have to read this buffer for reconstruct_write */ 2486 if (!test_bit(R5_OVERWRITE, &dev->flags) && i != sh->pd_idx && 2487 !test_bit(R5_LOCKED, &dev->flags) && 2488 !(test_bit(R5_UPTODATE, &dev->flags) || 2489 test_bit(R5_Wantcompute, &dev->flags))) { 2490 if (test_bit(R5_Insync, &dev->flags)) rcw++; 2491 else 2492 rcw += 2*disks; 2493 } 2494 } 2495 pr_debug("for sector %llu, rmw=%d rcw=%d\n", 2496 (unsigned long long)sh->sector, rmw, rcw); 2497 set_bit(STRIPE_HANDLE, &sh->state); 2498 if (rmw < rcw && rmw > 0) 2499 /* prefer read-modify-write, but need to get some data */ 2500 for (i = disks; i--; ) { 2501 struct r5dev *dev = &sh->dev[i]; 2502 if ((dev->towrite || i == sh->pd_idx) && 2503 !test_bit(R5_LOCKED, &dev->flags) && 2504 !(test_bit(R5_UPTODATE, &dev->flags) || 2505 test_bit(R5_Wantcompute, &dev->flags)) && 2506 test_bit(R5_Insync, &dev->flags)) { 2507 if ( 2508 test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 2509 pr_debug("Read_old block " 2510 "%d for r-m-w\n", i); 2511 set_bit(R5_LOCKED, &dev->flags); 2512 set_bit(R5_Wantread, &dev->flags); 2513 s->locked++; 2514 } else { 2515 set_bit(STRIPE_DELAYED, &sh->state); 2516 set_bit(STRIPE_HANDLE, &sh->state); 2517 } 2518 } 2519 } 2520 if (rcw <= rmw && rcw > 0) 2521 /* want reconstruct write, but need to get some data */ 2522 for (i = disks; i--; ) { 2523 struct r5dev *dev = &sh->dev[i]; 2524 if (!test_bit(R5_OVERWRITE, &dev->flags) && 2525 i != sh->pd_idx && 2526 !test_bit(R5_LOCKED, &dev->flags) && 2527 !(test_bit(R5_UPTODATE, &dev->flags) || 2528 test_bit(R5_Wantcompute, &dev->flags)) && 2529 test_bit(R5_Insync, &dev->flags)) { 2530 if ( 2531 test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 2532 pr_debug("Read_old block " 2533 "%d for Reconstruct\n", i); 2534 set_bit(R5_LOCKED, &dev->flags); 2535 set_bit(R5_Wantread, &dev->flags); 2536 s->locked++; 2537 } else { 2538 set_bit(STRIPE_DELAYED, &sh->state); 2539 set_bit(STRIPE_HANDLE, &sh->state); 2540 } 2541 } 2542 } 2543 /* now if nothing is locked, and if we have enough data, 2544 * we can start a write request 2545 */ 2546 /* since handle_stripe can be called at any time we need to handle the 2547 * case where a compute block operation has been submitted and then a 2548 * subsequent call wants to start a write request. raid_run_ops only 2549 * handles the case where compute block and reconstruct are requested 2550 * simultaneously. If this is not the case then new writes need to be 2551 * held off until the compute completes. 2552 */ 2553 if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) && 2554 (s->locked == 0 && (rcw == 0 || rmw == 0) && 2555 !test_bit(STRIPE_BIT_DELAY, &sh->state))) 2556 schedule_reconstruction(sh, s, rcw == 0, 0); 2557} 2558 2559static void handle_stripe_dirtying6(raid5_conf_t *conf, 2560 struct stripe_head *sh, struct stripe_head_state *s, 2561 struct r6_state *r6s, int disks) 2562{ 2563 int rcw = 0, pd_idx = sh->pd_idx, i; 2564 int qd_idx = sh->qd_idx; 2565 2566 set_bit(STRIPE_HANDLE, &sh->state); 2567 for (i = disks; i--; ) { 2568 struct r5dev *dev = &sh->dev[i]; 2569 /* check if we haven't enough data */ 2570 if (!test_bit(R5_OVERWRITE, &dev->flags) && 2571 i != pd_idx && i != qd_idx && 2572 !test_bit(R5_LOCKED, &dev->flags) && 2573 !(test_bit(R5_UPTODATE, &dev->flags) || 2574 test_bit(R5_Wantcompute, &dev->flags))) { 2575 rcw++; 2576 if (!test_bit(R5_Insync, &dev->flags)) 2577 continue; /* it's a failed drive */ 2578 2579 if ( 2580 test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 2581 pr_debug("Read_old stripe %llu " 2582 "block %d for Reconstruct\n", 2583 (unsigned long long)sh->sector, i); 2584 set_bit(R5_LOCKED, &dev->flags); 2585 set_bit(R5_Wantread, &dev->flags); 2586 s->locked++; 2587 } else { 2588 pr_debug("Request delayed stripe %llu " 2589 "block %d for Reconstruct\n", 2590 (unsigned long long)sh->sector, i); 2591 set_bit(STRIPE_DELAYED, &sh->state); 2592 set_bit(STRIPE_HANDLE, &sh->state); 2593 } 2594 } 2595 } 2596 /* now if nothing is locked, and if we have enough data, we can start a 2597 * write request 2598 */ 2599 if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) && 2600 s->locked == 0 && rcw == 0 && 2601 !test_bit(STRIPE_BIT_DELAY, &sh->state)) { 2602 schedule_reconstruction(sh, s, 1, 0); 2603 } 2604} 2605 2606static void handle_parity_checks5(raid5_conf_t *conf, struct stripe_head *sh, 2607 struct stripe_head_state *s, int disks) 2608{ 2609 struct r5dev *dev = NULL; 2610 2611 set_bit(STRIPE_HANDLE, &sh->state); 2612 2613 switch (sh->check_state) { 2614 case check_state_idle: 2615 /* start a new check operation if there are no failures */ 2616 if (s->failed == 0) { 2617 BUG_ON(s->uptodate != disks); 2618 sh->check_state = check_state_run; 2619 set_bit(STRIPE_OP_CHECK, &s->ops_request); 2620 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); 2621 s->uptodate--; 2622 break; 2623 } 2624 dev = &sh->dev[s->failed_num]; 2625 /* fall through */ 2626 case check_state_compute_result: 2627 sh->check_state = check_state_idle; 2628 if (!dev) 2629 dev = &sh->dev[sh->pd_idx]; 2630 2631 /* check that a write has not made the stripe insync */ 2632 if (test_bit(STRIPE_INSYNC, &sh->state)) 2633 break; 2634 2635 /* either failed parity check, or recovery is happening */ 2636 BUG_ON(!test_bit(R5_UPTODATE, &dev->flags)); 2637 BUG_ON(s->uptodate != disks); 2638 2639 set_bit(R5_LOCKED, &dev->flags); 2640 s->locked++; 2641 set_bit(R5_Wantwrite, &dev->flags); 2642 2643 clear_bit(STRIPE_DEGRADED, &sh->state); 2644 set_bit(STRIPE_INSYNC, &sh->state); 2645 break; 2646 case check_state_run: 2647 break; /* we will be called again upon completion */ 2648 case check_state_check_result: 2649 sh->check_state = check_state_idle; 2650 2651 /* if a failure occurred during the check operation, leave 2652 * STRIPE_INSYNC not set and let the stripe be handled again 2653 */ 2654 if (s->failed) 2655 break; 2656 2657 /* handle a successful check operation, if parity is correct 2658 * we are done. Otherwise update the mismatch count and repair 2659 * parity if !MD_RECOVERY_CHECK 2660 */ 2661 if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0) 2662 /* parity is correct (on disc, 2663 * not in buffer any more) 2664 */ 2665 set_bit(STRIPE_INSYNC, &sh->state); 2666 else { 2667 conf->mddev->resync_mismatches += STRIPE_SECTORS; 2668 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) 2669 /* don't try to repair!! */ 2670 set_bit(STRIPE_INSYNC, &sh->state); 2671 else { 2672 sh->check_state = check_state_compute_run; 2673 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2674 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2675 set_bit(R5_Wantcompute, 2676 &sh->dev[sh->pd_idx].flags); 2677 sh->ops.target = sh->pd_idx; 2678 sh->ops.target2 = -1; 2679 s->uptodate++; 2680 } 2681 } 2682 break; 2683 case check_state_compute_run: 2684 break; 2685 default: 2686 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", 2687 __func__, sh->check_state, 2688 (unsigned long long) sh->sector); 2689 BUG(); 2690 } 2691} 2692 2693 2694static void handle_parity_checks6(raid5_conf_t *conf, struct stripe_head *sh, 2695 struct stripe_head_state *s, 2696 struct r6_state *r6s, int disks) 2697{ 2698 int pd_idx = sh->pd_idx; 2699 int qd_idx = sh->qd_idx; 2700 struct r5dev *dev; 2701 2702 set_bit(STRIPE_HANDLE, &sh->state); 2703 2704 BUG_ON(s->failed > 2); 2705 2706 /* Want to check and possibly repair P and Q. 2707 * However there could be one 'failed' device, in which 2708 * case we can only check one of them, possibly using the 2709 * other to generate missing data 2710 */ 2711 2712 switch (sh->check_state) { 2713 case check_state_idle: 2714 /* start a new check operation if there are < 2 failures */ 2715 if (s->failed == r6s->q_failed) { 2716 /* The only possible failed device holds Q, so it 2717 * makes sense to check P (If anything else were failed, 2718 * we would have used P to recreate it). 2719 */ 2720 sh->check_state = check_state_run; 2721 } 2722 if (!r6s->q_failed && s->failed < 2) { 2723 /* Q is not failed, and we didn't use it to generate 2724 * anything, so it makes sense to check it 2725 */ 2726 if (sh->check_state == check_state_run) 2727 sh->check_state = check_state_run_pq; 2728 else 2729 sh->check_state = check_state_run_q; 2730 } 2731 2732 /* discard potentially stale zero_sum_result */ 2733 sh->ops.zero_sum_result = 0; 2734 2735 if (sh->check_state == check_state_run) { 2736 /* async_xor_zero_sum destroys the contents of P */ 2737 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 2738 s->uptodate--; 2739 } 2740 if (sh->check_state >= check_state_run && 2741 sh->check_state <= check_state_run_pq) { 2742 /* async_syndrome_zero_sum preserves P and Q, so 2743 * no need to mark them !uptodate here 2744 */ 2745 set_bit(STRIPE_OP_CHECK, &s->ops_request); 2746 break; 2747 } 2748 2749 /* we have 2-disk failure */ 2750 BUG_ON(s->failed != 2); 2751 /* fall through */ 2752 case check_state_compute_result: 2753 sh->check_state = check_state_idle; 2754 2755 /* check that a write has not made the stripe insync */ 2756 if (test_bit(STRIPE_INSYNC, &sh->state)) 2757 break; 2758 2759 /* now write out any block on a failed drive, 2760 * or P or Q if they were recomputed 2761 */ 2762 BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */ 2763 if (s->failed == 2) { 2764 dev = &sh->dev[r6s->failed_num[1]]; 2765 s->locked++; 2766 set_bit(R5_LOCKED, &dev->flags); 2767 set_bit(R5_Wantwrite, &dev->flags); 2768 } 2769 if (s->failed >= 1) { 2770 dev = &sh->dev[r6s->failed_num[0]]; 2771 s->locked++; 2772 set_bit(R5_LOCKED, &dev->flags); 2773 set_bit(R5_Wantwrite, &dev->flags); 2774 } 2775 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { 2776 dev = &sh->dev[pd_idx]; 2777 s->locked++; 2778 set_bit(R5_LOCKED, &dev->flags); 2779 set_bit(R5_Wantwrite, &dev->flags); 2780 } 2781 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { 2782 dev = &sh->dev[qd_idx]; 2783 s->locked++; 2784 set_bit(R5_LOCKED, &dev->flags); 2785 set_bit(R5_Wantwrite, &dev->flags); 2786 } 2787 clear_bit(STRIPE_DEGRADED, &sh->state); 2788 2789 set_bit(STRIPE_INSYNC, &sh->state); 2790 break; 2791 case check_state_run: 2792 case check_state_run_q: 2793 case check_state_run_pq: 2794 break; /* we will be called again upon completion */ 2795 case check_state_check_result: 2796 sh->check_state = check_state_idle; 2797 2798 /* handle a successful check operation, if parity is correct 2799 * we are done. Otherwise update the mismatch count and repair 2800 * parity if !MD_RECOVERY_CHECK 2801 */ 2802 if (sh->ops.zero_sum_result == 0) { 2803 /* both parities are correct */ 2804 if (!s->failed) 2805 set_bit(STRIPE_INSYNC, &sh->state); 2806 else { 2807 /* in contrast to the raid5 case we can validate 2808 * parity, but still have a failure to write 2809 * back 2810 */ 2811 sh->check_state = check_state_compute_result; 2812 /* Returning at this point means that we may go 2813 * off and bring p and/or q uptodate again so 2814 * we make sure to check zero_sum_result again 2815 * to verify if p or q need writeback 2816 */ 2817 } 2818 } else { 2819 conf->mddev->resync_mismatches += STRIPE_SECTORS; 2820 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) 2821 /* don't try to repair!! */ 2822 set_bit(STRIPE_INSYNC, &sh->state); 2823 else { 2824 int *target = &sh->ops.target; 2825 2826 sh->ops.target = -1; 2827 sh->ops.target2 = -1; 2828 sh->check_state = check_state_compute_run; 2829 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2830 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2831 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { 2832 set_bit(R5_Wantcompute, 2833 &sh->dev[pd_idx].flags); 2834 *target = pd_idx; 2835 target = &sh->ops.target2; 2836 s->uptodate++; 2837 } 2838 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { 2839 set_bit(R5_Wantcompute, 2840 &sh->dev[qd_idx].flags); 2841 *target = qd_idx; 2842 s->uptodate++; 2843 } 2844 } 2845 } 2846 break; 2847 case check_state_compute_run: 2848 break; 2849 default: 2850 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", 2851 __func__, sh->check_state, 2852 (unsigned long long) sh->sector); 2853 BUG(); 2854 } 2855} 2856 2857static void handle_stripe_expansion(raid5_conf_t *conf, struct stripe_head *sh, 2858 struct r6_state *r6s) 2859{ 2860 int i; 2861 2862 /* We have read all the blocks in this stripe and now we need to 2863 * copy some of them into a target stripe for expand. 2864 */ 2865 struct dma_async_tx_descriptor *tx = NULL; 2866 clear_bit(STRIPE_EXPAND_SOURCE, &sh->state); 2867 for (i = 0; i < sh->disks; i++) 2868 if (i != sh->pd_idx && i != sh->qd_idx) { 2869 int dd_idx, j; 2870 struct stripe_head *sh2; 2871 struct async_submit_ctl submit; 2872 2873 sector_t bn = compute_blocknr(sh, i, 1); 2874 sector_t s = raid5_compute_sector(conf, bn, 0, 2875 &dd_idx, NULL); 2876 sh2 = get_active_stripe(conf, s, 0, 1, 1); 2877 if (sh2 == NULL) 2878 /* so far only the early blocks of this stripe 2879 * have been requested. When later blocks 2880 * get requested, we will try again 2881 */ 2882 continue; 2883 if (!test_bit(STRIPE_EXPANDING, &sh2->state) || 2884 test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) { 2885 /* must have already done this block */ 2886 release_stripe(sh2); 2887 continue; 2888 } 2889 2890 /* place all the copies on one channel */ 2891 init_async_submit(&submit, 0, tx, NULL, NULL, NULL); 2892 tx = async_memcpy(sh2->dev[dd_idx].page, 2893 sh->dev[i].page, 0, 0, STRIPE_SIZE, 2894 &submit); 2895 2896 set_bit(R5_Expanded, &sh2->dev[dd_idx].flags); 2897 set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags); 2898 for (j = 0; j < conf->raid_disks; j++) 2899 if (j != sh2->pd_idx && 2900 (!r6s || j != sh2->qd_idx) && 2901 !test_bit(R5_Expanded, &sh2->dev[j].flags)) 2902 break; 2903 if (j == conf->raid_disks) { 2904 set_bit(STRIPE_EXPAND_READY, &sh2->state); 2905 set_bit(STRIPE_HANDLE, &sh2->state); 2906 } 2907 release_stripe(sh2); 2908 2909 } 2910 /* done submitting copies, wait for them to complete */ 2911 if (tx) { 2912 async_tx_ack(tx); 2913 dma_wait_for_async_tx(tx); 2914 } 2915} 2916 2917 2918/* 2919 * handle_stripe - do things to a stripe. 2920 * 2921 * We lock the stripe and then examine the state of various bits 2922 * to see what needs to be done. 2923 * Possible results: 2924 * return some read request which now have data 2925 * return some write requests which are safely on disc 2926 * schedule a read on some buffers 2927 * schedule a write of some buffers 2928 * return confirmation of parity correctness 2929 * 2930 * buffers are taken off read_list or write_list, and bh_cache buffers 2931 * get BH_Lock set before the stripe lock is released. 2932 * 2933 */ 2934 2935static void handle_stripe5(struct stripe_head *sh) 2936{ 2937 raid5_conf_t *conf = sh->raid_conf; 2938 int disks = sh->disks, i; 2939 struct bio *return_bi = NULL; 2940 struct stripe_head_state s; 2941 struct r5dev *dev; 2942 mdk_rdev_t *blocked_rdev = NULL; 2943 int prexor; 2944 2945 memset(&s, 0, sizeof(s)); 2946 pr_debug("handling stripe %llu, state=%#lx cnt=%d, pd_idx=%d check:%d " 2947 "reconstruct:%d\n", (unsigned long long)sh->sector, sh->state, 2948 atomic_read(&sh->count), sh->pd_idx, sh->check_state, 2949 sh->reconstruct_state); 2950 2951 spin_lock(&sh->lock); 2952 clear_bit(STRIPE_HANDLE, &sh->state); 2953 clear_bit(STRIPE_DELAYED, &sh->state); 2954 2955 s.syncing = test_bit(STRIPE_SYNCING, &sh->state); 2956 s.expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state); 2957 s.expanded = test_bit(STRIPE_EXPAND_READY, &sh->state); 2958 2959 /* Now to look around and see what can be done */ 2960 rcu_read_lock(); 2961 for (i=disks; i--; ) { 2962 mdk_rdev_t *rdev; 2963 2964 dev = &sh->dev[i]; 2965 clear_bit(R5_Insync, &dev->flags); 2966 2967 pr_debug("check %d: state 0x%lx toread %p read %p write %p " 2968 "written %p\n", i, dev->flags, dev->toread, dev->read, 2969 dev->towrite, dev->written); 2970 2971 /* maybe we can request a biofill operation 2972 * 2973 * new wantfill requests are only permitted while 2974 * ops_complete_biofill is guaranteed to be inactive 2975 */ 2976 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread && 2977 !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) 2978 set_bit(R5_Wantfill, &dev->flags); 2979 2980 /* now count some things */ 2981 if (test_bit(R5_LOCKED, &dev->flags)) s.locked++; 2982 if (test_bit(R5_UPTODATE, &dev->flags)) s.uptodate++; 2983 if (test_bit(R5_Wantcompute, &dev->flags)) s.compute++; 2984 2985 if (test_bit(R5_Wantfill, &dev->flags)) 2986 s.to_fill++; 2987 else if (dev->toread) 2988 s.to_read++; 2989 if (dev->towrite) { 2990 s.to_write++; 2991 if (!test_bit(R5_OVERWRITE, &dev->flags)) 2992 s.non_overwrite++; 2993 } 2994 if (dev->written) 2995 s.written++; 2996 rdev = rcu_dereference(conf->disks[i].rdev); 2997 if (blocked_rdev == NULL && 2998 rdev && unlikely(test_bit(Blocked, &rdev->flags))) { 2999 blocked_rdev = rdev; 3000 atomic_inc(&rdev->nr_pending); 3001 } 3002 if (!rdev || !test_bit(In_sync, &rdev->flags)) { 3003 /* The ReadError flag will just be confusing now */ 3004 clear_bit(R5_ReadError, &dev->flags); 3005 clear_bit(R5_ReWrite, &dev->flags); 3006 } 3007 if (!rdev || !test_bit(In_sync, &rdev->flags) 3008 || test_bit(R5_ReadError, &dev->flags)) { 3009 s.failed++; 3010 s.failed_num = i; 3011 } else 3012 set_bit(R5_Insync, &dev->flags); 3013 } 3014 rcu_read_unlock(); 3015 3016 if (unlikely(blocked_rdev)) { 3017 if (s.syncing || s.expanding || s.expanded || 3018 s.to_write || s.written) { 3019 set_bit(STRIPE_HANDLE, &sh->state); 3020 goto unlock; 3021 } 3022 /* There is nothing for the blocked_rdev to block */ 3023 rdev_dec_pending(blocked_rdev, conf->mddev); 3024 blocked_rdev = NULL; 3025 } 3026 3027 if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) { 3028 set_bit(STRIPE_OP_BIOFILL, &s.ops_request); 3029 set_bit(STRIPE_BIOFILL_RUN, &sh->state); 3030 } 3031 3032 pr_debug("locked=%d uptodate=%d to_read=%d" 3033 " to_write=%d failed=%d failed_num=%d\n", 3034 s.locked, s.uptodate, s.to_read, s.to_write, 3035 s.failed, s.failed_num); 3036 /* check if the array has lost two devices and, if so, some requests might 3037 * need to be failed 3038 */ 3039 if (s.failed > 1 && s.to_read+s.to_write+s.written) 3040 handle_failed_stripe(conf, sh, &s, disks, &return_bi); 3041 if (s.failed > 1 && s.syncing) { 3042 md_done_sync(conf->mddev, STRIPE_SECTORS,0); 3043 clear_bit(STRIPE_SYNCING, &sh->state); 3044 s.syncing = 0; 3045 } 3046 3047 /* might be able to return some write requests if the parity block 3048 * is safe, or on a failed drive 3049 */ 3050 dev = &sh->dev[sh->pd_idx]; 3051 if ( s.written && 3052 ((test_bit(R5_Insync, &dev->flags) && 3053 !test_bit(R5_LOCKED, &dev->flags) && 3054 test_bit(R5_UPTODATE, &dev->flags)) || 3055 (s.failed == 1 && s.failed_num == sh->pd_idx))) 3056 handle_stripe_clean_event(conf, sh, disks, &return_bi); 3057 3058 /* Now we might consider reading some blocks, either to check/generate 3059 * parity, or to satisfy requests 3060 * or to load a block that is being partially written. 3061 */ 3062 if (s.to_read || s.non_overwrite || 3063 (s.syncing && (s.uptodate + s.compute < disks)) || s.expanding) 3064 handle_stripe_fill5(sh, &s, disks); 3065 3066 /* Now we check to see if any write operations have recently 3067 * completed 3068 */ 3069 prexor = 0; 3070 if (sh->reconstruct_state == reconstruct_state_prexor_drain_result) 3071 prexor = 1; 3072 if (sh->reconstruct_state == reconstruct_state_drain_result || 3073 sh->reconstruct_state == reconstruct_state_prexor_drain_result) { 3074 sh->reconstruct_state = reconstruct_state_idle; 3075 3076 /* All the 'written' buffers and the parity block are ready to 3077 * be written back to disk 3078 */ 3079 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags)); 3080 for (i = disks; i--; ) { 3081 dev = &sh->dev[i]; 3082 if (test_bit(R5_LOCKED, &dev->flags) && 3083 (i == sh->pd_idx || dev->written)) { 3084 pr_debug("Writing block %d\n", i); 3085 set_bit(R5_Wantwrite, &dev->flags); 3086 if (prexor) 3087 continue; 3088 if (!test_bit(R5_Insync, &dev->flags) || 3089 (i == sh->pd_idx && s.failed == 0)) 3090 set_bit(STRIPE_INSYNC, &sh->state); 3091 } 3092 } 3093 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 3094 atomic_dec(&conf->preread_active_stripes); 3095 if (atomic_read(&conf->preread_active_stripes) < 3096 IO_THRESHOLD) 3097 md_wakeup_thread(conf->mddev->thread); 3098 } 3099 } 3100 3101 /* Now to consider new write requests and what else, if anything 3102 * should be read. We do not handle new writes when: 3103 * 1/ A 'write' operation (copy+xor) is already in flight. 3104 * 2/ A 'check' operation is in flight, as it may clobber the parity 3105 * block. 3106 */ 3107 if (s.to_write && !sh->reconstruct_state && !sh->check_state) 3108 handle_stripe_dirtying5(conf, sh, &s, disks); 3109 3110 /* maybe we need to check and possibly fix the parity for this stripe 3111 * Any reads will already have been scheduled, so we just see if enough 3112 * data is available. The parity check is held off while parity 3113 * dependent operations are in flight. 3114 */ 3115 if (sh->check_state || 3116 (s.syncing && s.locked == 0 && 3117 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && 3118 !test_bit(STRIPE_INSYNC, &sh->state))) 3119 handle_parity_checks5(conf, sh, &s, disks); 3120 3121 if (s.syncing && s.locked == 0 && test_bit(STRIPE_INSYNC, &sh->state)) { 3122 md_done_sync(conf->mddev, STRIPE_SECTORS,1); 3123 clear_bit(STRIPE_SYNCING, &sh->state); 3124 } 3125 3126 /* If the failed drive is just a ReadError, then we might need to progress 3127 * the repair/check process 3128 */ 3129 if (s.failed == 1 && !conf->mddev->ro && 3130 test_bit(R5_ReadError, &sh->dev[s.failed_num].flags) 3131 && !test_bit(R5_LOCKED, &sh->dev[s.failed_num].flags) 3132 && test_bit(R5_UPTODATE, &sh->dev[s.failed_num].flags) 3133 ) { 3134 dev = &sh->dev[s.failed_num]; 3135 if (!test_bit(R5_ReWrite, &dev->flags)) { 3136 set_bit(R5_Wantwrite, &dev->flags); 3137 set_bit(R5_ReWrite, &dev->flags); 3138 set_bit(R5_LOCKED, &dev->flags); 3139 s.locked++; 3140 } else { 3141 /* let's read it back */ 3142 set_bit(R5_Wantread, &dev->flags); 3143 set_bit(R5_LOCKED, &dev->flags); 3144 s.locked++; 3145 } 3146 } 3147 3148 /* Finish reconstruct operations initiated by the expansion process */ 3149 if (sh->reconstruct_state == reconstruct_state_result) { 3150 struct stripe_head *sh2 3151 = get_active_stripe(conf, sh->sector, 1, 1, 1); 3152 if (sh2 && test_bit(STRIPE_EXPAND_SOURCE, &sh2->state)) { 3153 /* sh cannot be written until sh2 has been read. 3154 * so arrange for sh to be delayed a little 3155 */ 3156 set_bit(STRIPE_DELAYED, &sh->state); 3157 set_bit(STRIPE_HANDLE, &sh->state); 3158 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, 3159 &sh2->state)) 3160 atomic_inc(&conf->preread_active_stripes); 3161 release_stripe(sh2); 3162 goto unlock; 3163 } 3164 if (sh2) 3165 release_stripe(sh2); 3166 3167 sh->reconstruct_state = reconstruct_state_idle; 3168 clear_bit(STRIPE_EXPANDING, &sh->state); 3169 for (i = conf->raid_disks; i--; ) { 3170 set_bit(R5_Wantwrite, &sh->dev[i].flags); 3171 set_bit(R5_LOCKED, &sh->dev[i].flags); 3172 s.locked++; 3173 } 3174 } 3175 3176 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) && 3177 !sh->reconstruct_state) { 3178 /* Need to write out all blocks after computing parity */ 3179 sh->disks = conf->raid_disks; 3180 stripe_set_idx(sh->sector, conf, 0, sh); 3181 schedule_reconstruction(sh, &s, 1, 1); 3182 } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) { 3183 clear_bit(STRIPE_EXPAND_READY, &sh->state); 3184 atomic_dec(&conf->reshape_stripes); 3185 wake_up(&conf->wait_for_overlap); 3186 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); 3187 } 3188 3189 if (s.expanding && s.locked == 0 && 3190 !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) 3191 handle_stripe_expansion(conf, sh, NULL); 3192 3193 unlock: 3194 spin_unlock(&sh->lock); 3195 3196 /* wait for this device to become unblocked */ 3197 if (unlikely(blocked_rdev)) 3198 md_wait_for_blocked_rdev(blocked_rdev, conf->mddev); 3199 3200 if (s.ops_request) 3201 raid_run_ops(sh, s.ops_request); 3202 3203 ops_run_io(sh, &s); 3204 3205 return_io(return_bi); 3206} 3207 3208static void handle_stripe6(struct stripe_head *sh) 3209{ 3210 raid5_conf_t *conf = sh->raid_conf; 3211 int disks = sh->disks; 3212 struct bio *return_bi = NULL; 3213 int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx; 3214 struct stripe_head_state s; 3215 struct r6_state r6s; 3216 struct r5dev *dev, *pdev, *qdev; 3217 mdk_rdev_t *blocked_rdev = NULL; 3218 3219 pr_debug("handling stripe %llu, state=%#lx cnt=%d, " 3220 "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n", 3221 (unsigned long long)sh->sector, sh->state, 3222 atomic_read(&sh->count), pd_idx, qd_idx, 3223 sh->check_state, sh->reconstruct_state); 3224 memset(&s, 0, sizeof(s)); 3225 3226 spin_lock(&sh->lock); 3227 clear_bit(STRIPE_HANDLE, &sh->state); 3228 clear_bit(STRIPE_DELAYED, &sh->state); 3229 3230 s.syncing = test_bit(STRIPE_SYNCING, &sh->state); 3231 s.expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state); 3232 s.expanded = test_bit(STRIPE_EXPAND_READY, &sh->state); 3233 /* Now to look around and see what can be done */ 3234 3235 rcu_read_lock(); 3236 for (i=disks; i--; ) { 3237 mdk_rdev_t *rdev; 3238 dev = &sh->dev[i]; 3239 clear_bit(R5_Insync, &dev->flags); 3240 3241 pr_debug("check %d: state 0x%lx read %p write %p written %p\n", 3242 i, dev->flags, dev->toread, dev->towrite, dev->written); 3243 /* maybe we can reply to a read 3244 * 3245 * new wantfill requests are only permitted while 3246 * ops_complete_biofill is guaranteed to be inactive 3247 */ 3248 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread && 3249 !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) 3250 set_bit(R5_Wantfill, &dev->flags); 3251 3252 /* now count some things */ 3253 if (test_bit(R5_LOCKED, &dev->flags)) s.locked++; 3254 if (test_bit(R5_UPTODATE, &dev->flags)) s.uptodate++; 3255 if (test_bit(R5_Wantcompute, &dev->flags)) { 3256 s.compute++; 3257 BUG_ON(s.compute > 2); 3258 } 3259 3260 if (test_bit(R5_Wantfill, &dev->flags)) { 3261 s.to_fill++; 3262 } else if (dev->toread) 3263 s.to_read++; 3264 if (dev->towrite) { 3265 s.to_write++; 3266 if (!test_bit(R5_OVERWRITE, &dev->flags)) 3267 s.non_overwrite++; 3268 } 3269 if (dev->written) 3270 s.written++; 3271 rdev = rcu_dereference(conf->disks[i].rdev); 3272 if (blocked_rdev == NULL && 3273 rdev && unlikely(test_bit(Blocked, &rdev->flags))) { 3274 blocked_rdev = rdev; 3275 atomic_inc(&rdev->nr_pending); 3276 } 3277 if (!rdev || !test_bit(In_sync, &rdev->flags)) { 3278 /* The ReadError flag will just be confusing now */ 3279 clear_bit(R5_ReadError, &dev->flags); 3280 clear_bit(R5_ReWrite, &dev->flags); 3281 } 3282 if (!rdev || !test_bit(In_sync, &rdev->flags) 3283 || test_bit(R5_ReadError, &dev->flags)) { 3284 if (s.failed < 2) 3285 r6s.failed_num[s.failed] = i; 3286 s.failed++; 3287 } else 3288 set_bit(R5_Insync, &dev->flags); 3289 } 3290 rcu_read_unlock(); 3291 3292 if (unlikely(blocked_rdev)) { 3293 if (s.syncing || s.expanding || s.expanded || 3294 s.to_write || s.written) { 3295 set_bit(STRIPE_HANDLE, &sh->state); 3296 goto unlock; 3297 } 3298 /* There is nothing for the blocked_rdev to block */ 3299 rdev_dec_pending(blocked_rdev, conf->mddev); 3300 blocked_rdev = NULL; 3301 } 3302 3303 if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) { 3304 set_bit(STRIPE_OP_BIOFILL, &s.ops_request); 3305 set_bit(STRIPE_BIOFILL_RUN, &sh->state); 3306 } 3307 3308 pr_debug("locked=%d uptodate=%d to_read=%d" 3309 " to_write=%d failed=%d failed_num=%d,%d\n", 3310 s.locked, s.uptodate, s.to_read, s.to_write, s.failed, 3311 r6s.failed_num[0], r6s.failed_num[1]); 3312 /* check if the array has lost >2 devices and, if so, some requests 3313 * might need to be failed 3314 */ 3315 if (s.failed > 2 && s.to_read+s.to_write+s.written) 3316 handle_failed_stripe(conf, sh, &s, disks, &return_bi); 3317 if (s.failed > 2 && s.syncing) { 3318 md_done_sync(conf->mddev, STRIPE_SECTORS,0); 3319 clear_bit(STRIPE_SYNCING, &sh->state); 3320 s.syncing = 0; 3321 } 3322 3323 /* 3324 * might be able to return some write requests if the parity blocks 3325 * are safe, or on a failed drive 3326 */ 3327 pdev = &sh->dev[pd_idx]; 3328 r6s.p_failed = (s.failed >= 1 && r6s.failed_num[0] == pd_idx) 3329 || (s.failed >= 2 && r6s.failed_num[1] == pd_idx); 3330 qdev = &sh->dev[qd_idx]; 3331 r6s.q_failed = (s.failed >= 1 && r6s.failed_num[0] == qd_idx) 3332 || (s.failed >= 2 && r6s.failed_num[1] == qd_idx); 3333 3334 if ( s.written && 3335 ( r6s.p_failed || ((test_bit(R5_Insync, &pdev->flags) 3336 && !test_bit(R5_LOCKED, &pdev->flags) 3337 && test_bit(R5_UPTODATE, &pdev->flags)))) && 3338 ( r6s.q_failed || ((test_bit(R5_Insync, &qdev->flags) 3339 && !test_bit(R5_LOCKED, &qdev->flags) 3340 && test_bit(R5_UPTODATE, &qdev->flags))))) 3341 handle_stripe_clean_event(conf, sh, disks, &return_bi); 3342 3343 /* Now we might consider reading some blocks, either to check/generate 3344 * parity, or to satisfy requests 3345 * or to load a block that is being partially written. 3346 */ 3347 if (s.to_read || s.non_overwrite || (s.to_write && s.failed) || 3348 (s.syncing && (s.uptodate + s.compute < disks)) || s.expanding) 3349 handle_stripe_fill6(sh, &s, &r6s, disks); 3350 3351 /* Now we check to see if any write operations have recently 3352 * completed 3353 */ 3354 if (sh->reconstruct_state == reconstruct_state_drain_result) { 3355 int qd_idx = sh->qd_idx; 3356 3357 sh->reconstruct_state = reconstruct_state_idle; 3358 /* All the 'written' buffers and the parity blocks are ready to 3359 * be written back to disk 3360 */ 3361 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags)); 3362 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags)); 3363 for (i = disks; i--; ) { 3364 dev = &sh->dev[i]; 3365 if (test_bit(R5_LOCKED, &dev->flags) && 3366 (i == sh->pd_idx || i == qd_idx || 3367 dev->written)) { 3368 pr_debug("Writing block %d\n", i); 3369 BUG_ON(!test_bit(R5_UPTODATE, &dev->flags)); 3370 set_bit(R5_Wantwrite, &dev->flags); 3371 if (!test_bit(R5_Insync, &dev->flags) || 3372 ((i == sh->pd_idx || i == qd_idx) && 3373 s.failed == 0)) 3374 set_bit(STRIPE_INSYNC, &sh->state); 3375 } 3376 } 3377 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 3378 atomic_dec(&conf->preread_active_stripes); 3379 if (atomic_read(&conf->preread_active_stripes) < 3380 IO_THRESHOLD) 3381 md_wakeup_thread(conf->mddev->thread); 3382 } 3383 } 3384 3385 /* Now to consider new write requests and what else, if anything 3386 * should be read. We do not handle new writes when: 3387 * 1/ A 'write' operation (copy+gen_syndrome) is already in flight. 3388 * 2/ A 'check' operation is in flight, as it may clobber the parity 3389 * block. 3390 */ 3391 if (s.to_write && !sh->reconstruct_state && !sh->check_state) 3392 handle_stripe_dirtying6(conf, sh, &s, &r6s, disks); 3393 3394 /* maybe we need to check and possibly fix the parity for this stripe 3395 * Any reads will already have been scheduled, so we just see if enough 3396 * data is available. The parity check is held off while parity 3397 * dependent operations are in flight. 3398 */ 3399 if (sh->check_state || 3400 (s.syncing && s.locked == 0 && 3401 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && 3402 !test_bit(STRIPE_INSYNC, &sh->state))) 3403 handle_parity_checks6(conf, sh, &s, &r6s, disks); 3404 3405 if (s.syncing && s.locked == 0 && test_bit(STRIPE_INSYNC, &sh->state)) { 3406 md_done_sync(conf->mddev, STRIPE_SECTORS,1); 3407 clear_bit(STRIPE_SYNCING, &sh->state); 3408 } 3409 3410 /* If the failed drives are just a ReadError, then we might need 3411 * to progress the repair/check process 3412 */ 3413 if (s.failed <= 2 && !conf->mddev->ro) 3414 for (i = 0; i < s.failed; i++) { 3415 dev = &sh->dev[r6s.failed_num[i]]; 3416 if (test_bit(R5_ReadError, &dev->flags) 3417 && !test_bit(R5_LOCKED, &dev->flags) 3418 && test_bit(R5_UPTODATE, &dev->flags) 3419 ) { 3420 if (!test_bit(R5_ReWrite, &dev->flags)) { 3421 set_bit(R5_Wantwrite, &dev->flags); 3422 set_bit(R5_ReWrite, &dev->flags); 3423 set_bit(R5_LOCKED, &dev->flags); 3424 s.locked++; 3425 } else { 3426 /* let's read it back */ 3427 set_bit(R5_Wantread, &dev->flags); 3428 set_bit(R5_LOCKED, &dev->flags); 3429 s.locked++; 3430 } 3431 } 3432 } 3433 3434 /* Finish reconstruct operations initiated by the expansion process */ 3435 if (sh->reconstruct_state == reconstruct_state_result) { 3436 sh->reconstruct_state = reconstruct_state_idle; 3437 clear_bit(STRIPE_EXPANDING, &sh->state); 3438 for (i = conf->raid_disks; i--; ) { 3439 set_bit(R5_Wantwrite, &sh->dev[i].flags); 3440 set_bit(R5_LOCKED, &sh->dev[i].flags); 3441 s.locked++; 3442 } 3443 } 3444 3445 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) && 3446 !sh->reconstruct_state) { 3447 struct stripe_head *sh2 3448 = get_active_stripe(conf, sh->sector, 1, 1, 1); 3449 if (sh2 && test_bit(STRIPE_EXPAND_SOURCE, &sh2->state)) { 3450 /* sh cannot be written until sh2 has been read. 3451 * so arrange for sh to be delayed a little 3452 */ 3453 set_bit(STRIPE_DELAYED, &sh->state); 3454 set_bit(STRIPE_HANDLE, &sh->state); 3455 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, 3456 &sh2->state)) 3457 atomic_inc(&conf->preread_active_stripes); 3458 release_stripe(sh2); 3459 goto unlock; 3460 } 3461 if (sh2) 3462 release_stripe(sh2); 3463 3464 /* Need to write out all blocks after computing P&Q */ 3465 sh->disks = conf->raid_disks; 3466 stripe_set_idx(sh->sector, conf, 0, sh); 3467 schedule_reconstruction(sh, &s, 1, 1); 3468 } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) { 3469 clear_bit(STRIPE_EXPAND_READY, &sh->state); 3470 atomic_dec(&conf->reshape_stripes); 3471 wake_up(&conf->wait_for_overlap); 3472 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); 3473 } 3474 3475 if (s.expanding && s.locked == 0 && 3476 !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) 3477 handle_stripe_expansion(conf, sh, &r6s); 3478 3479 unlock: 3480 spin_unlock(&sh->lock); 3481 3482 /* wait for this device to become unblocked */ 3483 if (unlikely(blocked_rdev)) 3484 md_wait_for_blocked_rdev(blocked_rdev, conf->mddev); 3485 3486 if (s.ops_request) 3487 raid_run_ops(sh, s.ops_request); 3488 3489 ops_run_io(sh, &s); 3490 3491 return_io(return_bi); 3492} 3493 3494static void handle_stripe(struct stripe_head *sh) 3495{ 3496 if (sh->raid_conf->level == 6) 3497 handle_stripe6(sh); 3498 else 3499 handle_stripe5(sh); 3500} 3501 3502static void raid5_activate_delayed(raid5_conf_t *conf) 3503{ 3504 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) { 3505 while (!list_empty(&conf->delayed_list)) { 3506 struct list_head *l = conf->delayed_list.next; 3507 struct stripe_head *sh; 3508 sh = list_entry(l, struct stripe_head, lru); 3509 list_del_init(l); 3510 clear_bit(STRIPE_DELAYED, &sh->state); 3511 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 3512 atomic_inc(&conf->preread_active_stripes); 3513 list_add_tail(&sh->lru, &conf->hold_list); 3514 } 3515 } else 3516 blk_plug_device(conf->mddev->queue); 3517} 3518 3519static void activate_bit_delay(raid5_conf_t *conf) 3520{ 3521 /* device_lock is held */ 3522 struct list_head head; 3523 list_add(&head, &conf->bitmap_list); 3524 list_del_init(&conf->bitmap_list); 3525 while (!list_empty(&head)) { 3526 struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru); 3527 list_del_init(&sh->lru); 3528 atomic_inc(&sh->count); 3529 __release_stripe(conf, sh); 3530 } 3531} 3532 3533static void unplug_slaves(mddev_t *mddev) 3534{ 3535 raid5_conf_t *conf = mddev->private; 3536 int i; 3537 3538 rcu_read_lock(); 3539 for (i = 0; i < conf->raid_disks; i++) { 3540 mdk_rdev_t *rdev = rcu_dereference(conf->disks[i].rdev); 3541 if (rdev && !test_bit(Faulty, &rdev->flags) && atomic_read(&rdev->nr_pending)) { 3542 struct request_queue *r_queue = bdev_get_queue(rdev->bdev); 3543 3544 atomic_inc(&rdev->nr_pending); 3545 rcu_read_unlock(); 3546 3547 blk_unplug(r_queue); 3548 3549 rdev_dec_pending(rdev, mddev); 3550 rcu_read_lock(); 3551 } 3552 } 3553 rcu_read_unlock(); 3554} 3555 3556static void raid5_unplug_device(struct request_queue *q) 3557{ 3558 mddev_t *mddev = q->queuedata; 3559 raid5_conf_t *conf = mddev->private; 3560 unsigned long flags; 3561 3562 spin_lock_irqsave(&conf->device_lock, flags); 3563 3564 if (blk_remove_plug(q)) { 3565 conf->seq_flush++; 3566 raid5_activate_delayed(conf); 3567 } 3568 md_wakeup_thread(mddev->thread); 3569 3570 spin_unlock_irqrestore(&conf->device_lock, flags); 3571 3572 unplug_slaves(mddev); 3573} 3574 3575static int raid5_congested(void *data, int bits) 3576{ 3577 mddev_t *mddev = data; 3578 raid5_conf_t *conf = mddev->private; 3579 3580 /* No difference between reads and writes. Just check 3581 * how busy the stripe_cache is 3582 */ 3583 3584 if (mddev_congested(mddev, bits)) 3585 return 1; 3586 if (conf->inactive_blocked) 3587 return 1; 3588 if (conf->quiesce) 3589 return 1; 3590 if (list_empty_careful(&conf->inactive_list)) 3591 return 1; 3592 3593 return 0; 3594} 3595 3596/* We want read requests to align with chunks where possible, 3597 * but write requests don't need to. 3598 */ 3599static int raid5_mergeable_bvec(struct request_queue *q, 3600 struct bvec_merge_data *bvm, 3601 struct bio_vec *biovec) 3602{ 3603 mddev_t *mddev = q->queuedata; 3604 sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev); 3605 int max; 3606 unsigned int chunk_sectors = mddev->chunk_sectors; 3607 unsigned int bio_sectors = bvm->bi_size >> 9; 3608 3609 if ((bvm->bi_rw & 1) == WRITE) 3610 return biovec->bv_len; /* always allow writes to be mergeable */ 3611 3612 if (mddev->new_chunk_sectors < mddev->chunk_sectors) 3613 chunk_sectors = mddev->new_chunk_sectors; 3614 max = (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9; 3615 if (max < 0) max = 0; 3616 if (max <= biovec->bv_len && bio_sectors == 0) 3617 return biovec->bv_len; 3618 else 3619 return max; 3620} 3621 3622 3623static int in_chunk_boundary(mddev_t *mddev, struct bio *bio) 3624{ 3625 sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev); 3626 unsigned int chunk_sectors = mddev->chunk_sectors; 3627 unsigned int bio_sectors = bio->bi_size >> 9; 3628 3629 if (mddev->new_chunk_sectors < mddev->chunk_sectors) 3630 chunk_sectors = mddev->new_chunk_sectors; 3631 return chunk_sectors >= 3632 ((sector & (chunk_sectors - 1)) + bio_sectors); 3633} 3634 3635/* 3636 * add bio to the retry LIFO ( in O(1) ... we are in interrupt ) 3637 * later sampled by raid5d. 3638 */ 3639static void add_bio_to_retry(struct bio *bi,raid5_conf_t *conf) 3640{ 3641 unsigned long flags; 3642 3643 spin_lock_irqsave(&conf->device_lock, flags); 3644 3645 bi->bi_next = conf->retry_read_aligned_list; 3646 conf->retry_read_aligned_list = bi; 3647 3648 spin_unlock_irqrestore(&conf->device_lock, flags); 3649 md_wakeup_thread(conf->mddev->thread); 3650} 3651 3652 3653static struct bio *remove_bio_from_retry(raid5_conf_t *conf) 3654{ 3655 struct bio *bi; 3656 3657 bi = conf->retry_read_aligned; 3658 if (bi) { 3659 conf->retry_read_aligned = NULL; 3660 return bi; 3661 } 3662 bi = conf->retry_read_aligned_list; 3663 if(bi) { 3664 conf->retry_read_aligned_list = bi->bi_next; 3665 bi->bi_next = NULL; 3666 /* 3667 * this sets the active strip count to 1 and the processed 3668 * strip count to zero (upper 8 bits) 3669 */ 3670 bi->bi_phys_segments = 1; /* biased count of active stripes */ 3671 } 3672 3673 return bi; 3674} 3675 3676 3677/* 3678 * The "raid5_align_endio" should check if the read succeeded and if it 3679 * did, call bio_endio on the original bio (having bio_put the new bio 3680 * first). 3681 * If the read failed.. 3682 */ 3683static void raid5_align_endio(struct bio *bi, int error) 3684{ 3685 struct bio* raid_bi = bi->bi_private; 3686 mddev_t *mddev; 3687 raid5_conf_t *conf; 3688 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 3689 mdk_rdev_t *rdev; 3690 3691 bio_put(bi); 3692 3693 mddev = raid_bi->bi_bdev->bd_disk->queue->queuedata; 3694 conf = mddev->private; 3695 rdev = (void*)raid_bi->bi_next; 3696 raid_bi->bi_next = NULL; 3697 3698 rdev_dec_pending(rdev, conf->mddev); 3699 3700 if (!error && uptodate) { 3701 bio_endio(raid_bi, 0); 3702 if (atomic_dec_and_test(&conf->active_aligned_reads)) 3703 wake_up(&conf->wait_for_stripe); 3704 return; 3705 } 3706 3707 3708 pr_debug("raid5_align_endio : io error...handing IO for a retry\n"); 3709 3710 add_bio_to_retry(raid_bi, conf); 3711} 3712 3713static int bio_fits_rdev(struct bio *bi) 3714{ 3715 struct request_queue *q = bdev_get_queue(bi->bi_bdev); 3716 3717 if ((bi->bi_size>>9) > queue_max_sectors(q)) 3718 return 0; 3719 blk_recount_segments(q, bi); 3720 if (bi->bi_phys_segments > queue_max_phys_segments(q)) 3721 return 0; 3722 3723 if (q->merge_bvec_fn) 3724 /* it's too hard to apply the merge_bvec_fn at this stage, 3725 * just just give up 3726 */ 3727 return 0; 3728 3729 return 1; 3730} 3731 3732 3733static int chunk_aligned_read(struct request_queue *q, struct bio * raid_bio) 3734{ 3735 mddev_t *mddev = q->queuedata; 3736 raid5_conf_t *conf = mddev->private; 3737 unsigned int dd_idx; 3738 struct bio* align_bi; 3739 mdk_rdev_t *rdev; 3740 3741 if (!in_chunk_boundary(mddev, raid_bio)) { 3742 pr_debug("chunk_aligned_read : non aligned\n"); 3743 return 0; 3744 } 3745 /* 3746 * use bio_clone to make a copy of the bio 3747 */ 3748 align_bi = bio_clone(raid_bio, GFP_NOIO); 3749 if (!align_bi) 3750 return 0; 3751 /* 3752 * set bi_end_io to a new function, and set bi_private to the 3753 * original bio. 3754 */ 3755 align_bi->bi_end_io = raid5_align_endio; 3756 align_bi->bi_private = raid_bio; 3757 /* 3758 * compute position 3759 */ 3760 align_bi->bi_sector = raid5_compute_sector(conf, raid_bio->bi_sector, 3761 0, 3762 &dd_idx, NULL); 3763 3764 rcu_read_lock(); 3765 rdev = rcu_dereference(conf->disks[dd_idx].rdev); 3766 if (rdev && test_bit(In_sync, &rdev->flags)) { 3767 atomic_inc(&rdev->nr_pending); 3768 rcu_read_unlock(); 3769 raid_bio->bi_next = (void*)rdev; 3770 align_bi->bi_bdev = rdev->bdev; 3771 align_bi->bi_flags &= ~(1 << BIO_SEG_VALID); 3772 align_bi->bi_sector += rdev->data_offset; 3773 3774 if (!bio_fits_rdev(align_bi)) { 3775 /* too big in some way */ 3776 bio_put(align_bi); 3777 rdev_dec_pending(rdev, mddev); 3778 return 0; 3779 } 3780 3781 spin_lock_irq(&conf->device_lock); 3782 wait_event_lock_irq(conf->wait_for_stripe, 3783 conf->quiesce == 0, 3784 conf->device_lock, /* nothing */); 3785 atomic_inc(&conf->active_aligned_reads); 3786 spin_unlock_irq(&conf->device_lock); 3787 3788 generic_make_request(align_bi); 3789 return 1; 3790 } else { 3791 rcu_read_unlock(); 3792 bio_put(align_bi); 3793 return 0; 3794 } 3795} 3796 3797/* __get_priority_stripe - get the next stripe to process 3798 * 3799 * Full stripe writes are allowed to pass preread active stripes up until 3800 * the bypass_threshold is exceeded. In general the bypass_count 3801 * increments when the handle_list is handled before the hold_list; however, it 3802 * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a 3803 * stripe with in flight i/o. The bypass_count will be reset when the 3804 * head of the hold_list has changed, i.e. the head was promoted to the 3805 * handle_list. 3806 */ 3807static struct stripe_head *__get_priority_stripe(raid5_conf_t *conf) 3808{ 3809 struct stripe_head *sh; 3810 3811 pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n", 3812 __func__, 3813 list_empty(&conf->handle_list) ? "empty" : "busy", 3814 list_empty(&conf->hold_list) ? "empty" : "busy", 3815 atomic_read(&conf->pending_full_writes), conf->bypass_count); 3816 3817 if (!list_empty(&conf->handle_list)) { 3818 sh = list_entry(conf->handle_list.next, typeof(*sh), lru); 3819 3820 if (list_empty(&conf->hold_list)) 3821 conf->bypass_count = 0; 3822 else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) { 3823 if (conf->hold_list.next == conf->last_hold) 3824 conf->bypass_count++; 3825 else { 3826 conf->last_hold = conf->hold_list.next; 3827 conf->bypass_count -= conf->bypass_threshold; 3828 if (conf->bypass_count < 0) 3829 conf->bypass_count = 0; 3830 } 3831 } 3832 } else if (!list_empty(&conf->hold_list) && 3833 ((conf->bypass_threshold && 3834 conf->bypass_count > conf->bypass_threshold) || 3835 atomic_read(&conf->pending_full_writes) == 0)) { 3836 sh = list_entry(conf->hold_list.next, 3837 typeof(*sh), lru); 3838 conf->bypass_count -= conf->bypass_threshold; 3839 if (conf->bypass_count < 0) 3840 conf->bypass_count = 0; 3841 } else 3842 return NULL; 3843 3844 list_del_init(&sh->lru); 3845 atomic_inc(&sh->count); 3846 BUG_ON(atomic_read(&sh->count) != 1); 3847 return sh; 3848} 3849 3850static int make_request(struct request_queue *q, struct bio * bi) 3851{ 3852 mddev_t *mddev = q->queuedata; 3853 raid5_conf_t *conf = mddev->private; 3854 int dd_idx; 3855 sector_t new_sector; 3856 sector_t logical_sector, last_sector; 3857 struct stripe_head *sh; 3858 const int rw = bio_data_dir(bi); 3859 int cpu, remaining; 3860 3861 if (unlikely(bio_rw_flagged(bi, BIO_RW_BARRIER))) { 3862 bio_endio(bi, -EOPNOTSUPP); 3863 return 0; 3864 } 3865 3866 md_write_start(mddev, bi); 3867 3868 cpu = part_stat_lock(); 3869 part_stat_inc(cpu, &mddev->gendisk->part0, ios[rw]); 3870 part_stat_add(cpu, &mddev->gendisk->part0, sectors[rw], 3871 bio_sectors(bi)); 3872 part_stat_unlock(); 3873 3874 if (rw == READ && 3875 mddev->reshape_position == MaxSector && 3876 chunk_aligned_read(q,bi)) 3877 return 0; 3878 3879 logical_sector = bi->bi_sector & ~((sector_t)STRIPE_SECTORS-1); 3880 last_sector = bi->bi_sector + (bi->bi_size>>9); 3881 bi->bi_next = NULL; 3882 bi->bi_phys_segments = 1; /* over-loaded to count active stripes */ 3883 3884 for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) { 3885 DEFINE_WAIT(w); 3886 int disks, data_disks; 3887 int previous; 3888 3889 retry: 3890 previous = 0; 3891 disks = conf->raid_disks; 3892 prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE); 3893 if (unlikely(conf->reshape_progress != MaxSector)) { 3894 /* spinlock is needed as reshape_progress may be 3895 * 64bit on a 32bit platform, and so it might be 3896 * possible to see a half-updated value 3897 * Ofcourse reshape_progress could change after 3898 * the lock is dropped, so once we get a reference 3899 * to the stripe that we think it is, we will have 3900 * to check again. 3901 */ 3902 spin_lock_irq(&conf->device_lock); 3903 if (mddev->delta_disks < 0 3904 ? logical_sector < conf->reshape_progress 3905 : logical_sector >= conf->reshape_progress) { 3906 disks = conf->previous_raid_disks; 3907 previous = 1; 3908 } else { 3909 if (mddev->delta_disks < 0 3910 ? logical_sector < conf->reshape_safe 3911 : logical_sector >= conf->reshape_safe) { 3912 spin_unlock_irq(&conf->device_lock); 3913 schedule(); 3914 goto retry; 3915 } 3916 } 3917 spin_unlock_irq(&conf->device_lock); 3918 } 3919 data_disks = disks - conf->max_degraded; 3920 3921 new_sector = raid5_compute_sector(conf, logical_sector, 3922 previous, 3923 &dd_idx, NULL); 3924 pr_debug("raid5: make_request, sector %llu logical %llu\n", 3925 (unsigned long long)new_sector, 3926 (unsigned long long)logical_sector); 3927 3928 sh = get_active_stripe(conf, new_sector, previous, 3929 (bi->bi_rw&RWA_MASK), 0); 3930 if (sh) { 3931 if (unlikely(previous)) { 3932 /* expansion might have moved on while waiting for a 3933 * stripe, so we must do the range check again. 3934 * Expansion could still move past after this 3935 * test, but as we are holding a reference to 3936 * 'sh', we know that if that happens, 3937 * STRIPE_EXPANDING will get set and the expansion 3938 * won't proceed until we finish with the stripe. 3939 */ 3940 int must_retry = 0; 3941 spin_lock_irq(&conf->device_lock); 3942 if (mddev->delta_disks < 0 3943 ? logical_sector >= conf->reshape_progress 3944 : logical_sector < conf->reshape_progress) 3945 /* mismatch, need to try again */ 3946 must_retry = 1; 3947 spin_unlock_irq(&conf->device_lock); 3948 if (must_retry) { 3949 release_stripe(sh); 3950 schedule(); 3951 goto retry; 3952 } 3953 } 3954 3955 if (bio_data_dir(bi) == WRITE && 3956 logical_sector >= mddev->suspend_lo && 3957 logical_sector < mddev->suspend_hi) { 3958 release_stripe(sh); 3959 /* As the suspend_* range is controlled by 3960 * userspace, we want an interruptible 3961 * wait. 3962 */ 3963 flush_signals(current); 3964 prepare_to_wait(&conf->wait_for_overlap, 3965 &w, TASK_INTERRUPTIBLE); 3966 if (logical_sector >= mddev->suspend_lo && 3967 logical_sector < mddev->suspend_hi) 3968 schedule(); 3969 goto retry; 3970 } 3971 3972 if (test_bit(STRIPE_EXPANDING, &sh->state) || 3973 !add_stripe_bio(sh, bi, dd_idx, (bi->bi_rw&RW_MASK))) { 3974 /* Stripe is busy expanding or 3975 * add failed due to overlap. Flush everything 3976 * and wait a while 3977 */ 3978 raid5_unplug_device(mddev->queue); 3979 release_stripe(sh); 3980 schedule(); 3981 goto retry; 3982 } 3983 finish_wait(&conf->wait_for_overlap, &w); 3984 set_bit(STRIPE_HANDLE, &sh->state); 3985 clear_bit(STRIPE_DELAYED, &sh->state); 3986 release_stripe(sh); 3987 } else { 3988 /* cannot get stripe for read-ahead, just give-up */ 3989 clear_bit(BIO_UPTODATE, &bi->bi_flags); 3990 finish_wait(&conf->wait_for_overlap, &w); 3991 break; 3992 } 3993 3994 } 3995 spin_lock_irq(&conf->device_lock); 3996 remaining = raid5_dec_bi_phys_segments(bi); 3997 spin_unlock_irq(&conf->device_lock); 3998 if (remaining == 0) { 3999 4000 if ( rw == WRITE ) 4001 md_write_end(mddev); 4002 4003 bio_endio(bi, 0); 4004 } 4005 return 0; 4006} 4007 4008static sector_t raid5_size(mddev_t *mddev, sector_t sectors, int raid_disks); 4009 4010static sector_t reshape_request(mddev_t *mddev, sector_t sector_nr, int *skipped) 4011{ 4012 /* reshaping is quite different to recovery/resync so it is 4013 * handled quite separately ... here. 4014 * 4015 * On each call to sync_request, we gather one chunk worth of 4016 * destination stripes and flag them as expanding. 4017 * Then we find all the source stripes and request reads. 4018 * As the reads complete, handle_stripe will copy the data 4019 * into the destination stripe and release that stripe. 4020 */ 4021 raid5_conf_t *conf = (raid5_conf_t *) mddev->private; 4022 struct stripe_head *sh; 4023 sector_t first_sector, last_sector; 4024 int raid_disks = conf->previous_raid_disks; 4025 int data_disks = raid_disks - conf->max_degraded; 4026 int new_data_disks = conf->raid_disks - conf->max_degraded; 4027 int i; 4028 int dd_idx; 4029 sector_t writepos, readpos, safepos; 4030 sector_t stripe_addr; 4031 int reshape_sectors; 4032 struct list_head stripes; 4033 4034 if (sector_nr == 0) { 4035 /* If restarting in the middle, skip the initial sectors */ 4036 if (mddev->delta_disks < 0 && 4037 conf->reshape_progress < raid5_size(mddev, 0, 0)) { 4038 sector_nr = raid5_size(mddev, 0, 0) 4039 - conf->reshape_progress; 4040 } else if (mddev->delta_disks >= 0 && 4041 conf->reshape_progress > 0) 4042 sector_nr = conf->reshape_progress; 4043 sector_div(sector_nr, new_data_disks); 4044 if (sector_nr) { 4045 *skipped = 1; 4046 return sector_nr; 4047 } 4048 } 4049 4050 /* We need to process a full chunk at a time. 4051 * If old and new chunk sizes differ, we need to process the 4052 * largest of these 4053 */ 4054 if (mddev->new_chunk_sectors > mddev->chunk_sectors) 4055 reshape_sectors = mddev->new_chunk_sectors; 4056 else 4057 reshape_sectors = mddev->chunk_sectors; 4058 4059 /* we update the metadata when there is more than 3Meg 4060 * in the block range (that is rather arbitrary, should 4061 * probably be time based) or when the data about to be 4062 * copied would over-write the source of the data at 4063 * the front of the range. 4064 * i.e. one new_stripe along from reshape_progress new_maps 4065 * to after where reshape_safe old_maps to 4066 */ 4067 writepos = conf->reshape_progress; 4068 sector_div(writepos, new_data_disks); 4069 readpos = conf->reshape_progress; 4070 sector_div(readpos, data_disks); 4071 safepos = conf->reshape_safe; 4072 sector_div(safepos, data_disks); 4073 if (mddev->delta_disks < 0) { 4074 writepos -= min_t(sector_t, reshape_sectors, writepos); 4075 readpos += reshape_sectors; 4076 safepos += reshape_sectors; 4077 } else { 4078 writepos += reshape_sectors; 4079 readpos -= min_t(sector_t, reshape_sectors, readpos); 4080 safepos -= min_t(sector_t, reshape_sectors, safepos); 4081 } 4082 4083 /* 'writepos' is the most advanced device address we might write. 4084 * 'readpos' is the least advanced device address we might read. 4085 * 'safepos' is the least address recorded in the metadata as having 4086 * been reshaped. 4087 * If 'readpos' is behind 'writepos', then there is no way that we can 4088 * ensure safety in the face of a crash - that must be done by userspace 4089 * making a backup of the data. So in that case there is no particular 4090 * rush to update metadata. 4091 * Otherwise if 'safepos' is behind 'writepos', then we really need to 4092 * update the metadata to advance 'safepos' to match 'readpos' so that 4093 * we can be safe in the event of a crash. 4094 * So we insist on updating metadata if safepos is behind writepos and 4095 * readpos is beyond writepos. 4096 * In any case, update the metadata every 10 seconds. 4097 * Maybe that number should be configurable, but I'm not sure it is 4098 * worth it.... maybe it could be a multiple of safemode_delay??? 4099 */ 4100 if ((mddev->delta_disks < 0 4101 ? (safepos > writepos && readpos < writepos) 4102 : (safepos < writepos && readpos > writepos)) || 4103 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { 4104 /* Cannot proceed until we've updated the superblock... */ 4105 wait_event(conf->wait_for_overlap, 4106 atomic_read(&conf->reshape_stripes)==0); 4107 mddev->reshape_position = conf->reshape_progress; 4108 mddev->curr_resync_completed = mddev->curr_resync; 4109 conf->reshape_checkpoint = jiffies; 4110 set_bit(MD_CHANGE_DEVS, &mddev->flags); 4111 md_wakeup_thread(mddev->thread); 4112 wait_event(mddev->sb_wait, mddev->flags == 0 || 4113 kthread_should_stop()); 4114 spin_lock_irq(&conf->device_lock); 4115 conf->reshape_safe = mddev->reshape_position; 4116 spin_unlock_irq(&conf->device_lock); 4117 wake_up(&conf->wait_for_overlap); 4118 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4119 } 4120 4121 if (mddev->delta_disks < 0) { 4122 BUG_ON(conf->reshape_progress == 0); 4123 stripe_addr = writepos; 4124 BUG_ON((mddev->dev_sectors & 4125 ~((sector_t)reshape_sectors - 1)) 4126 - reshape_sectors - stripe_addr 4127 != sector_nr); 4128 } else { 4129 BUG_ON(writepos != sector_nr + reshape_sectors); 4130 stripe_addr = sector_nr; 4131 } 4132 INIT_LIST_HEAD(&stripes); 4133 for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) { 4134 int j; 4135 int skipped_disk = 0; 4136 sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1); 4137 set_bit(STRIPE_EXPANDING, &sh->state); 4138 atomic_inc(&conf->reshape_stripes); 4139 /* If any of this stripe is beyond the end of the old 4140 * array, then we need to zero those blocks 4141 */ 4142 for (j=sh->disks; j--;) { 4143 sector_t s; 4144 if (j == sh->pd_idx) 4145 continue; 4146 if (conf->level == 6 && 4147 j == sh->qd_idx) 4148 continue; 4149 s = compute_blocknr(sh, j, 0); 4150 if (s < raid5_size(mddev, 0, 0)) { 4151 skipped_disk = 1; 4152 continue; 4153 } 4154 memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE); 4155 set_bit(R5_Expanded, &sh->dev[j].flags); 4156 set_bit(R5_UPTODATE, &sh->dev[j].flags); 4157 } 4158 if (!skipped_disk) { 4159 set_bit(STRIPE_EXPAND_READY, &sh->state); 4160 set_bit(STRIPE_HANDLE, &sh->state); 4161 } 4162 list_add(&sh->lru, &stripes); 4163 } 4164 spin_lock_irq(&conf->device_lock); 4165 if (mddev->delta_disks < 0) 4166 conf->reshape_progress -= reshape_sectors * new_data_disks; 4167 else 4168 conf->reshape_progress += reshape_sectors * new_data_disks; 4169 spin_unlock_irq(&conf->device_lock); 4170 /* Ok, those stripe are ready. We can start scheduling 4171 * reads on the source stripes. 4172 * The source stripes are determined by mapping the first and last 4173 * block on the destination stripes. 4174 */ 4175 first_sector = 4176 raid5_compute_sector(conf, stripe_addr*(new_data_disks), 4177 1, &dd_idx, NULL); 4178 last_sector = 4179 raid5_compute_sector(conf, ((stripe_addr+reshape_sectors) 4180 * new_data_disks - 1), 4181 1, &dd_idx, NULL); 4182 if (last_sector >= mddev->dev_sectors) 4183 last_sector = mddev->dev_sectors - 1; 4184 while (first_sector <= last_sector) { 4185 sh = get_active_stripe(conf, first_sector, 1, 0, 1); 4186 set_bit(STRIPE_EXPAND_SOURCE, &sh->state); 4187 set_bit(STRIPE_HANDLE, &sh->state); 4188 release_stripe(sh); 4189 first_sector += STRIPE_SECTORS; 4190 } 4191 /* Now that the sources are clearly marked, we can release 4192 * the destination stripes 4193 */ 4194 while (!list_empty(&stripes)) { 4195 sh = list_entry(stripes.next, struct stripe_head, lru); 4196 list_del_init(&sh->lru); 4197 release_stripe(sh); 4198 } 4199 /* If this takes us to the resync_max point where we have to pause, 4200 * then we need to write out the superblock. 4201 */ 4202 sector_nr += reshape_sectors; 4203 if ((sector_nr - mddev->curr_resync_completed) * 2 4204 >= mddev->resync_max - mddev->curr_resync_completed) { 4205 /* Cannot proceed until we've updated the superblock... */ 4206 wait_event(conf->wait_for_overlap, 4207 atomic_read(&conf->reshape_stripes) == 0); 4208 mddev->reshape_position = conf->reshape_progress; 4209 mddev->curr_resync_completed = mddev->curr_resync + reshape_sectors; 4210 conf->reshape_checkpoint = jiffies; 4211 set_bit(MD_CHANGE_DEVS, &mddev->flags); 4212 md_wakeup_thread(mddev->thread); 4213 wait_event(mddev->sb_wait, 4214 !test_bit(MD_CHANGE_DEVS, &mddev->flags) 4215 || kthread_should_stop()); 4216 spin_lock_irq(&conf->device_lock); 4217 conf->reshape_safe = mddev->reshape_position; 4218 spin_unlock_irq(&conf->device_lock); 4219 wake_up(&conf->wait_for_overlap); 4220 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4221 } 4222 return reshape_sectors; 4223} 4224 4225/* FIXME go_faster isn't used */ 4226static inline sector_t sync_request(mddev_t *mddev, sector_t sector_nr, int *skipped, int go_faster) 4227{ 4228 raid5_conf_t *conf = (raid5_conf_t *) mddev->private; 4229 struct stripe_head *sh; 4230 sector_t max_sector = mddev->dev_sectors; 4231 int sync_blocks; 4232 int still_degraded = 0; 4233 int i; 4234 4235 if (sector_nr >= max_sector) { 4236 /* just being told to finish up .. nothing much to do */ 4237 unplug_slaves(mddev); 4238 4239 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) { 4240 end_reshape(conf); 4241 return 0; 4242 } 4243 4244 if (mddev->curr_resync < max_sector) /* aborted */ 4245 bitmap_end_sync(mddev->bitmap, mddev->curr_resync, 4246 &sync_blocks, 1); 4247 else /* completed sync */ 4248 conf->fullsync = 0; 4249 bitmap_close_sync(mddev->bitmap); 4250 4251 return 0; 4252 } 4253 4254 /* Allow raid5_quiesce to complete */ 4255 wait_event(conf->wait_for_overlap, conf->quiesce != 2); 4256 4257 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 4258 return reshape_request(mddev, sector_nr, skipped); 4259 4260 /* No need to check resync_max as we never do more than one 4261 * stripe, and as resync_max will always be on a chunk boundary, 4262 * if the check in md_do_sync didn't fire, there is no chance 4263 * of overstepping resync_max here 4264 */ 4265 4266 /* if there is too many failed drives and we are trying 4267 * to resync, then assert that we are finished, because there is 4268 * nothing we can do. 4269 */ 4270 if (mddev->degraded >= conf->max_degraded && 4271 test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 4272 sector_t rv = mddev->dev_sectors - sector_nr; 4273 *skipped = 1; 4274 return rv; 4275 } 4276 if (!bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) && 4277 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && 4278 !conf->fullsync && sync_blocks >= STRIPE_SECTORS) { 4279 /* we can skip this block, and probably more */ 4280 sync_blocks /= STRIPE_SECTORS; 4281 *skipped = 1; 4282 return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */ 4283 } 4284 4285 4286 bitmap_cond_end_sync(mddev->bitmap, sector_nr); 4287 4288 sh = get_active_stripe(conf, sector_nr, 0, 1, 0); 4289 if (sh == NULL) { 4290 sh = get_active_stripe(conf, sector_nr, 0, 0, 0); 4291 /* make sure we don't swamp the stripe cache if someone else 4292 * is trying to get access 4293 */ 4294 schedule_timeout_uninterruptible(1); 4295 } 4296 /* Need to check if array will still be degraded after recovery/resync 4297 * We don't need to check the 'failed' flag as when that gets set, 4298 * recovery aborts. 4299 */ 4300 for (i = 0; i < conf->raid_disks; i++) 4301 if (conf->disks[i].rdev == NULL) 4302 still_degraded = 1; 4303 4304 bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded); 4305 4306 spin_lock(&sh->lock); 4307 set_bit(STRIPE_SYNCING, &sh->state); 4308 clear_bit(STRIPE_INSYNC, &sh->state); 4309 spin_unlock(&sh->lock); 4310 4311 handle_stripe(sh); 4312 release_stripe(sh); 4313 4314 return STRIPE_SECTORS; 4315} 4316 4317static int retry_aligned_read(raid5_conf_t *conf, struct bio *raid_bio) 4318{ 4319 /* We may not be able to submit a whole bio at once as there 4320 * may not be enough stripe_heads available. 4321 * We cannot pre-allocate enough stripe_heads as we may need 4322 * more than exist in the cache (if we allow ever large chunks). 4323 * So we do one stripe head at a time and record in 4324 * ->bi_hw_segments how many have been done. 4325 * 4326 * We *know* that this entire raid_bio is in one chunk, so 4327 * it will be only one 'dd_idx' and only need one call to raid5_compute_sector. 4328 */ 4329 struct stripe_head *sh; 4330 int dd_idx; 4331 sector_t sector, logical_sector, last_sector; 4332 int scnt = 0; 4333 int remaining; 4334 int handled = 0; 4335 4336 logical_sector = raid_bio->bi_sector & ~((sector_t)STRIPE_SECTORS-1); 4337 sector = raid5_compute_sector(conf, logical_sector, 4338 0, &dd_idx, NULL); 4339 last_sector = raid_bio->bi_sector + (raid_bio->bi_size>>9); 4340 4341 for (; logical_sector < last_sector; 4342 logical_sector += STRIPE_SECTORS, 4343 sector += STRIPE_SECTORS, 4344 scnt++) { 4345 4346 if (scnt < raid5_bi_hw_segments(raid_bio)) 4347 /* already done this stripe */ 4348 continue; 4349 4350 sh = get_active_stripe(conf, sector, 0, 1, 0); 4351 4352 if (!sh) { 4353 /* failed to get a stripe - must wait */ 4354 raid5_set_bi_hw_segments(raid_bio, scnt); 4355 conf->retry_read_aligned = raid_bio; 4356 return handled; 4357 } 4358 4359 set_bit(R5_ReadError, &sh->dev[dd_idx].flags); 4360 if (!add_stripe_bio(sh, raid_bio, dd_idx, 0)) { 4361 release_stripe(sh); 4362 raid5_set_bi_hw_segments(raid_bio, scnt); 4363 conf->retry_read_aligned = raid_bio; 4364 return handled; 4365 } 4366 4367 handle_stripe(sh); 4368 release_stripe(sh); 4369 handled++; 4370 } 4371 spin_lock_irq(&conf->device_lock); 4372 remaining = raid5_dec_bi_phys_segments(raid_bio); 4373 spin_unlock_irq(&conf->device_lock); 4374 if (remaining == 0) 4375 bio_endio(raid_bio, 0); 4376 if (atomic_dec_and_test(&conf->active_aligned_reads)) 4377 wake_up(&conf->wait_for_stripe); 4378 return handled; 4379} 4380 4381 4382/* 4383 * This is our raid5 kernel thread. 4384 * 4385 * We scan the hash table for stripes which can be handled now. 4386 * During the scan, completed stripes are saved for us by the interrupt 4387 * handler, so that they will not have to wait for our next wakeup. 4388 */ 4389static void raid5d(mddev_t *mddev) 4390{ 4391 struct stripe_head *sh; 4392 raid5_conf_t *conf = mddev->private; 4393 int handled; 4394 4395 pr_debug("+++ raid5d active\n"); 4396 4397 md_check_recovery(mddev); 4398 4399 handled = 0; 4400 spin_lock_irq(&conf->device_lock); 4401 while (1) { 4402 struct bio *bio; 4403 4404 if (conf->seq_flush != conf->seq_write) { 4405 int seq = conf->seq_flush; 4406 spin_unlock_irq(&conf->device_lock); 4407 bitmap_unplug(mddev->bitmap); 4408 spin_lock_irq(&conf->device_lock); 4409 conf->seq_write = seq; 4410 activate_bit_delay(conf); 4411 } 4412 4413 while ((bio = remove_bio_from_retry(conf))) { 4414 int ok; 4415 spin_unlock_irq(&conf->device_lock); 4416 ok = retry_aligned_read(conf, bio); 4417 spin_lock_irq(&conf->device_lock); 4418 if (!ok) 4419 break; 4420 handled++; 4421 } 4422 4423 sh = __get_priority_stripe(conf); 4424 4425 if (!sh) 4426 break; 4427 spin_unlock_irq(&conf->device_lock); 4428 4429 handled++; 4430 handle_stripe(sh); 4431 release_stripe(sh); 4432 cond_resched(); 4433 4434 spin_lock_irq(&conf->device_lock); 4435 } 4436 pr_debug("%d stripes handled\n", handled); 4437 4438 spin_unlock_irq(&conf->device_lock); 4439 4440 async_tx_issue_pending_all(); 4441 unplug_slaves(mddev); 4442 4443 pr_debug("--- raid5d inactive\n"); 4444} 4445 4446static ssize_t 4447raid5_show_stripe_cache_size(mddev_t *mddev, char *page) 4448{ 4449 raid5_conf_t *conf = mddev->private; 4450 if (conf) 4451 return sprintf(page, "%d\n", conf->max_nr_stripes); 4452 else 4453 return 0; 4454} 4455 4456static ssize_t 4457raid5_store_stripe_cache_size(mddev_t *mddev, const char *page, size_t len) 4458{ 4459 raid5_conf_t *conf = mddev->private; 4460 unsigned long new; 4461 int err; 4462 4463 if (len >= PAGE_SIZE) 4464 return -EINVAL; 4465 if (!conf) 4466 return -ENODEV; 4467 4468 if (strict_strtoul(page, 10, &new)) 4469 return -EINVAL; 4470 if (new <= 16 || new > 32768) 4471 return -EINVAL; 4472 while (new < conf->max_nr_stripes) { 4473 if (drop_one_stripe(conf)) 4474 conf->max_nr_stripes--; 4475 else 4476 break; 4477 } 4478 err = md_allow_write(mddev); 4479 if (err) 4480 return err; 4481 while (new > conf->max_nr_stripes) { 4482 if (grow_one_stripe(conf)) 4483 conf->max_nr_stripes++; 4484 else break; 4485 } 4486 return len; 4487} 4488 4489static struct md_sysfs_entry 4490raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR, 4491 raid5_show_stripe_cache_size, 4492 raid5_store_stripe_cache_size); 4493 4494static ssize_t 4495raid5_show_preread_threshold(mddev_t *mddev, char *page) 4496{ 4497 raid5_conf_t *conf = mddev->private; 4498 if (conf) 4499 return sprintf(page, "%d\n", conf->bypass_threshold); 4500 else 4501 return 0; 4502} 4503 4504static ssize_t 4505raid5_store_preread_threshold(mddev_t *mddev, const char *page, size_t len) 4506{ 4507 raid5_conf_t *conf = mddev->private; 4508 unsigned long new; 4509 if (len >= PAGE_SIZE) 4510 return -EINVAL; 4511 if (!conf) 4512 return -ENODEV; 4513 4514 if (strict_strtoul(page, 10, &new)) 4515 return -EINVAL; 4516 if (new > conf->max_nr_stripes) 4517 return -EINVAL; 4518 conf->bypass_threshold = new; 4519 return len; 4520} 4521 4522static struct md_sysfs_entry 4523raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold, 4524 S_IRUGO | S_IWUSR, 4525 raid5_show_preread_threshold, 4526 raid5_store_preread_threshold); 4527 4528static ssize_t 4529stripe_cache_active_show(mddev_t *mddev, char *page) 4530{ 4531 raid5_conf_t *conf = mddev->private; 4532 if (conf) 4533 return sprintf(page, "%d\n", atomic_read(&conf->active_stripes)); 4534 else 4535 return 0; 4536} 4537 4538static struct md_sysfs_entry 4539raid5_stripecache_active = __ATTR_RO(stripe_cache_active); 4540 4541static struct attribute *raid5_attrs[] = { 4542 &raid5_stripecache_size.attr, 4543 &raid5_stripecache_active.attr, 4544 &raid5_preread_bypass_threshold.attr, 4545 NULL, 4546}; 4547static struct attribute_group raid5_attrs_group = { 4548 .name = NULL, 4549 .attrs = raid5_attrs, 4550}; 4551 4552static sector_t 4553raid5_size(mddev_t *mddev, sector_t sectors, int raid_disks) 4554{ 4555 raid5_conf_t *conf = mddev->private; 4556 4557 if (!sectors) 4558 sectors = mddev->dev_sectors; 4559 if (!raid_disks) { 4560 /* size is defined by the smallest of previous and new size */ 4561 if (conf->raid_disks < conf->previous_raid_disks) 4562 raid_disks = conf->raid_disks; 4563 else 4564 raid_disks = conf->previous_raid_disks; 4565 } 4566 4567 sectors &= ~((sector_t)mddev->chunk_sectors - 1); 4568 sectors &= ~((sector_t)mddev->new_chunk_sectors - 1); 4569 return sectors * (raid_disks - conf->max_degraded); 4570} 4571 4572static void raid5_free_percpu(raid5_conf_t *conf) 4573{ 4574 struct raid5_percpu *percpu; 4575 unsigned long cpu; 4576 4577 if (!conf->percpu) 4578 return; 4579 4580 get_online_cpus(); 4581 for_each_possible_cpu(cpu) { 4582 percpu = per_cpu_ptr(conf->percpu, cpu); 4583 safe_put_page(percpu->spare_page); 4584 kfree(percpu->scribble); 4585 } 4586#ifdef CONFIG_HOTPLUG_CPU 4587 unregister_cpu_notifier(&conf->cpu_notify); 4588#endif 4589 put_online_cpus(); 4590 4591 free_percpu(conf->percpu); 4592} 4593 4594static void free_conf(raid5_conf_t *conf) 4595{ 4596 shrink_stripes(conf); 4597 raid5_free_percpu(conf); 4598 kfree(conf->disks); 4599 kfree(conf->stripe_hashtbl); 4600 kfree(conf); 4601} 4602 4603#ifdef CONFIG_HOTPLUG_CPU 4604static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action, 4605 void *hcpu) 4606{ 4607 raid5_conf_t *conf = container_of(nfb, raid5_conf_t, cpu_notify); 4608 long cpu = (long)hcpu; 4609 struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu); 4610 4611 switch (action) { 4612 case CPU_UP_PREPARE: 4613 case CPU_UP_PREPARE_FROZEN: 4614 if (conf->level == 6 && !percpu->spare_page) 4615 percpu->spare_page = alloc_page(GFP_KERNEL); 4616 if (!percpu->scribble) 4617 percpu->scribble = kmalloc(conf->scribble_len, GFP_KERNEL); 4618 4619 if (!percpu->scribble || 4620 (conf->level == 6 && !percpu->spare_page)) { 4621 safe_put_page(percpu->spare_page); 4622 kfree(percpu->scribble); 4623 pr_err("%s: failed memory allocation for cpu%ld\n", 4624 __func__, cpu); 4625 return NOTIFY_BAD; 4626 } 4627 break; 4628 case CPU_DEAD: 4629 case CPU_DEAD_FROZEN: 4630 safe_put_page(percpu->spare_page); 4631 kfree(percpu->scribble); 4632 percpu->spare_page = NULL; 4633 percpu->scribble = NULL; 4634 break; 4635 default: 4636 break; 4637 } 4638 return NOTIFY_OK; 4639} 4640#endif 4641 4642static int raid5_alloc_percpu(raid5_conf_t *conf) 4643{ 4644 unsigned long cpu; 4645 struct page *spare_page; 4646 struct raid5_percpu *allcpus; 4647 void *scribble; 4648 int err; 4649 4650 allcpus = alloc_percpu(struct raid5_percpu); 4651 if (!allcpus) 4652 return -ENOMEM; 4653 conf->percpu = allcpus; 4654 4655 get_online_cpus(); 4656 err = 0; 4657 for_each_present_cpu(cpu) { 4658 if (conf->level == 6) { 4659 spare_page = alloc_page(GFP_KERNEL); 4660 if (!spare_page) { 4661 err = -ENOMEM; 4662 break; 4663 } 4664 per_cpu_ptr(conf->percpu, cpu)->spare_page = spare_page; 4665 } 4666 scribble = kmalloc(scribble_len(conf->raid_disks), GFP_KERNEL); 4667 if (!scribble) { 4668 err = -ENOMEM; 4669 break; 4670 } 4671 per_cpu_ptr(conf->percpu, cpu)->scribble = scribble; 4672 } 4673#ifdef CONFIG_HOTPLUG_CPU 4674 conf->cpu_notify.notifier_call = raid456_cpu_notify; 4675 conf->cpu_notify.priority = 0; 4676 if (err == 0) 4677 err = register_cpu_notifier(&conf->cpu_notify); 4678#endif 4679 put_online_cpus(); 4680 4681 return err; 4682} 4683 4684static raid5_conf_t *setup_conf(mddev_t *mddev) 4685{ 4686 raid5_conf_t *conf; 4687 int raid_disk, memory; 4688 mdk_rdev_t *rdev; 4689 struct disk_info *disk; 4690 4691 if (mddev->new_level != 5 4692 && mddev->new_level != 4 4693 && mddev->new_level != 6) { 4694 printk(KERN_ERR "raid5: %s: raid level not set to 4/5/6 (%d)\n", 4695 mdname(mddev), mddev->new_level); 4696 return ERR_PTR(-EIO); 4697 } 4698 if ((mddev->new_level == 5 4699 && !algorithm_valid_raid5(mddev->new_layout)) || 4700 (mddev->new_level == 6 4701 && !algorithm_valid_raid6(mddev->new_layout))) { 4702 printk(KERN_ERR "raid5: %s: layout %d not supported\n", 4703 mdname(mddev), mddev->new_layout); 4704 return ERR_PTR(-EIO); 4705 } 4706 if (mddev->new_level == 6 && mddev->raid_disks < 4) { 4707 printk(KERN_ERR "raid6: not enough configured devices for %s (%d, minimum 4)\n", 4708 mdname(mddev), mddev->raid_disks); 4709 return ERR_PTR(-EINVAL); 4710 } 4711 4712 if (!mddev->new_chunk_sectors || 4713 (mddev->new_chunk_sectors << 9) % PAGE_SIZE || 4714 !is_power_of_2(mddev->new_chunk_sectors)) { 4715 printk(KERN_ERR "raid5: invalid chunk size %d for %s\n", 4716 mddev->new_chunk_sectors << 9, mdname(mddev)); 4717 return ERR_PTR(-EINVAL); 4718 } 4719 4720 conf = kzalloc(sizeof(raid5_conf_t), GFP_KERNEL); 4721 if (conf == NULL) 4722 goto abort; 4723 spin_lock_init(&conf->device_lock); 4724 init_waitqueue_head(&conf->wait_for_stripe); 4725 init_waitqueue_head(&conf->wait_for_overlap); 4726 INIT_LIST_HEAD(&conf->handle_list); 4727 INIT_LIST_HEAD(&conf->hold_list); 4728 INIT_LIST_HEAD(&conf->delayed_list); 4729 INIT_LIST_HEAD(&conf->bitmap_list); 4730 INIT_LIST_HEAD(&conf->inactive_list); 4731 atomic_set(&conf->active_stripes, 0); 4732 atomic_set(&conf->preread_active_stripes, 0); 4733 atomic_set(&conf->active_aligned_reads, 0); 4734 conf->bypass_threshold = BYPASS_THRESHOLD; 4735 4736 conf->raid_disks = mddev->raid_disks; 4737 conf->scribble_len = scribble_len(conf->raid_disks); 4738 if (mddev->reshape_position == MaxSector) 4739 conf->previous_raid_disks = mddev->raid_disks; 4740 else 4741 conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks; 4742 4743 conf->disks = kzalloc(conf->raid_disks * sizeof(struct disk_info), 4744 GFP_KERNEL); 4745 if (!conf->disks) 4746 goto abort; 4747 4748 conf->mddev = mddev; 4749 4750 if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL) 4751 goto abort; 4752 4753 conf->level = mddev->new_level; 4754 if (raid5_alloc_percpu(conf) != 0) 4755 goto abort; 4756 4757 pr_debug("raid5: run(%s) called.\n", mdname(mddev)); 4758 4759 list_for_each_entry(rdev, &mddev->disks, same_set) { 4760 raid_disk = rdev->raid_disk; 4761 if (raid_disk >= conf->raid_disks 4762 || raid_disk < 0) 4763 continue; 4764 disk = conf->disks + raid_disk; 4765 4766 disk->rdev = rdev; 4767 4768 if (test_bit(In_sync, &rdev->flags)) { 4769 char b[BDEVNAME_SIZE]; 4770 printk(KERN_INFO "raid5: device %s operational as raid" 4771 " disk %d\n", bdevname(rdev->bdev,b), 4772 raid_disk); 4773 } else 4774 /* Cannot rely on bitmap to complete recovery */ 4775 conf->fullsync = 1; 4776 } 4777 4778 conf->chunk_sectors = mddev->new_chunk_sectors; 4779 conf->level = mddev->new_level; 4780 if (conf->level == 6) 4781 conf->max_degraded = 2; 4782 else 4783 conf->max_degraded = 1; 4784 conf->algorithm = mddev->new_layout; 4785 conf->max_nr_stripes = NR_STRIPES; 4786 conf->reshape_progress = mddev->reshape_position; 4787 if (conf->reshape_progress != MaxSector) { 4788 conf->prev_chunk_sectors = mddev->chunk_sectors; 4789 conf->prev_algo = mddev->layout; 4790 } 4791 4792 memory = conf->max_nr_stripes * (sizeof(struct stripe_head) + 4793 conf->raid_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024; 4794 if (grow_stripes(conf, conf->max_nr_stripes)) { 4795 printk(KERN_ERR 4796 "raid5: couldn't allocate %dkB for buffers\n", memory); 4797 goto abort; 4798 } else 4799 printk(KERN_INFO "raid5: allocated %dkB for %s\n", 4800 memory, mdname(mddev)); 4801 4802 conf->thread = md_register_thread(raid5d, mddev, NULL); 4803 if (!conf->thread) { 4804 printk(KERN_ERR 4805 "raid5: couldn't allocate thread for %s\n", 4806 mdname(mddev)); 4807 goto abort; 4808 } 4809 4810 return conf; 4811 4812 abort: 4813 if (conf) { 4814 free_conf(conf); 4815 return ERR_PTR(-EIO); 4816 } else 4817 return ERR_PTR(-ENOMEM); 4818} 4819 4820static int run(mddev_t *mddev) 4821{ 4822 raid5_conf_t *conf; 4823 int working_disks = 0, chunk_size; 4824 mdk_rdev_t *rdev; 4825 4826 if (mddev->recovery_cp != MaxSector) 4827 printk(KERN_NOTICE "raid5: %s is not clean" 4828 " -- starting background reconstruction\n", 4829 mdname(mddev)); 4830 if (mddev->reshape_position != MaxSector) { 4831 /* Check that we can continue the reshape. 4832 * Currently only disks can change, it must 4833 * increase, and we must be past the point where 4834 * a stripe over-writes itself 4835 */ 4836 sector_t here_new, here_old; 4837 int old_disks; 4838 int max_degraded = (mddev->level == 6 ? 2 : 1); 4839 4840 if (mddev->new_level != mddev->level) { 4841 printk(KERN_ERR "raid5: %s: unsupported reshape " 4842 "required - aborting.\n", 4843 mdname(mddev)); 4844 return -EINVAL; 4845 } 4846 old_disks = mddev->raid_disks - mddev->delta_disks; 4847 /* reshape_position must be on a new-stripe boundary, and one 4848 * further up in new geometry must map after here in old 4849 * geometry. 4850 */ 4851 here_new = mddev->reshape_position; 4852 if (sector_div(here_new, mddev->new_chunk_sectors * 4853 (mddev->raid_disks - max_degraded))) { 4854 printk(KERN_ERR "raid5: reshape_position not " 4855 "on a stripe boundary\n"); 4856 return -EINVAL; 4857 } 4858 /* here_new is the stripe we will write to */ 4859 here_old = mddev->reshape_position; 4860 sector_div(here_old, mddev->chunk_sectors * 4861 (old_disks-max_degraded)); 4862 /* here_old is the first stripe that we might need to read 4863 * from */ 4864 if (mddev->delta_disks == 0) { 4865 /* We cannot be sure it is safe to start an in-place 4866 * reshape. It is only safe if user-space if monitoring 4867 * and taking constant backups. 4868 * mdadm always starts a situation like this in 4869 * readonly mode so it can take control before 4870 * allowing any writes. So just check for that. 4871 */ 4872 if ((here_new * mddev->new_chunk_sectors != 4873 here_old * mddev->chunk_sectors) || 4874 mddev->ro == 0) { 4875 printk(KERN_ERR "raid5: in-place reshape must be started" 4876 " in read-only mode - aborting\n"); 4877 return -EINVAL; 4878 } 4879 } else if (mddev->delta_disks < 0 4880 ? (here_new * mddev->new_chunk_sectors <= 4881 here_old * mddev->chunk_sectors) 4882 : (here_new * mddev->new_chunk_sectors >= 4883 here_old * mddev->chunk_sectors)) { 4884 /* Reading from the same stripe as writing to - bad */ 4885 printk(KERN_ERR "raid5: reshape_position too early for " 4886 "auto-recovery - aborting.\n"); 4887 return -EINVAL; 4888 } 4889 printk(KERN_INFO "raid5: reshape will continue\n"); 4890 /* OK, we should be able to continue; */ 4891 } else { 4892 BUG_ON(mddev->level != mddev->new_level); 4893 BUG_ON(mddev->layout != mddev->new_layout); 4894 BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors); 4895 BUG_ON(mddev->delta_disks != 0); 4896 } 4897 4898 if (mddev->private == NULL) 4899 conf = setup_conf(mddev); 4900 else 4901 conf = mddev->private; 4902 4903 if (IS_ERR(conf)) 4904 return PTR_ERR(conf); 4905 4906 mddev->thread = conf->thread; 4907 conf->thread = NULL; 4908 mddev->private = conf; 4909 4910 /* 4911 * 0 for a fully functional array, 1 or 2 for a degraded array. 4912 */ 4913 list_for_each_entry(rdev, &mddev->disks, same_set) 4914 if (rdev->raid_disk >= 0 && 4915 test_bit(In_sync, &rdev->flags)) 4916 working_disks++; 4917 4918 mddev->degraded = conf->raid_disks - working_disks; 4919 4920 if (mddev->degraded > conf->max_degraded) { 4921 printk(KERN_ERR "raid5: not enough operational devices for %s" 4922 " (%d/%d failed)\n", 4923 mdname(mddev), mddev->degraded, conf->raid_disks); 4924 goto abort; 4925 } 4926 4927 /* device size must be a multiple of chunk size */ 4928 mddev->dev_sectors &= ~(mddev->chunk_sectors - 1); 4929 mddev->resync_max_sectors = mddev->dev_sectors; 4930 4931 if (mddev->degraded > 0 && 4932 mddev->recovery_cp != MaxSector) { 4933 if (mddev->ok_start_degraded) 4934 printk(KERN_WARNING 4935 "raid5: starting dirty degraded array: %s" 4936 "- data corruption possible.\n", 4937 mdname(mddev)); 4938 else { 4939 printk(KERN_ERR 4940 "raid5: cannot start dirty degraded array for %s\n", 4941 mdname(mddev)); 4942 goto abort; 4943 } 4944 } 4945 4946 if (mddev->degraded == 0) 4947 printk("raid5: raid level %d set %s active with %d out of %d" 4948 " devices, algorithm %d\n", conf->level, mdname(mddev), 4949 mddev->raid_disks-mddev->degraded, mddev->raid_disks, 4950 mddev->new_layout); 4951 else 4952 printk(KERN_ALERT "raid5: raid level %d set %s active with %d" 4953 " out of %d devices, algorithm %d\n", conf->level, 4954 mdname(mddev), mddev->raid_disks - mddev->degraded, 4955 mddev->raid_disks, mddev->new_layout); 4956 4957 print_raid5_conf(conf); 4958 4959 if (conf->reshape_progress != MaxSector) { 4960 printk("...ok start reshape thread\n"); 4961 conf->reshape_safe = conf->reshape_progress; 4962 atomic_set(&conf->reshape_stripes, 0); 4963 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 4964 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 4965 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 4966 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 4967 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 4968 "reshape"); 4969 } 4970 4971 /* read-ahead size must cover two whole stripes, which is 4972 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices 4973 */ 4974 { 4975 int data_disks = conf->previous_raid_disks - conf->max_degraded; 4976 int stripe = data_disks * 4977 ((mddev->chunk_sectors << 9) / PAGE_SIZE); 4978 if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 4979 mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 4980 } 4981 4982 /* Ok, everything is just fine now */ 4983 if (sysfs_create_group(&mddev->kobj, &raid5_attrs_group)) 4984 printk(KERN_WARNING 4985 "raid5: failed to create sysfs attributes for %s\n", 4986 mdname(mddev)); 4987 4988 mddev->queue->queue_lock = &conf->device_lock; 4989 4990 mddev->queue->unplug_fn = raid5_unplug_device; 4991 mddev->queue->backing_dev_info.congested_data = mddev; 4992 mddev->queue->backing_dev_info.congested_fn = raid5_congested; 4993 4994 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); 4995 4996 blk_queue_merge_bvec(mddev->queue, raid5_mergeable_bvec); 4997 chunk_size = mddev->chunk_sectors << 9; 4998 blk_queue_io_min(mddev->queue, chunk_size); 4999 blk_queue_io_opt(mddev->queue, chunk_size * 5000 (conf->raid_disks - conf->max_degraded)); 5001 5002 list_for_each_entry(rdev, &mddev->disks, same_set) 5003 disk_stack_limits(mddev->gendisk, rdev->bdev, 5004 rdev->data_offset << 9); 5005 5006 return 0; 5007abort: 5008 md_unregister_thread(mddev->thread); 5009 mddev->thread = NULL; 5010 if (conf) { 5011 print_raid5_conf(conf); 5012 free_conf(conf); 5013 } 5014 mddev->private = NULL; 5015 printk(KERN_ALERT "raid5: failed to run raid set %s\n", mdname(mddev)); 5016 return -EIO; 5017} 5018 5019 5020 5021static int stop(mddev_t *mddev) 5022{ 5023 raid5_conf_t *conf = (raid5_conf_t *) mddev->private; 5024 5025 md_unregister_thread(mddev->thread); 5026 mddev->thread = NULL; 5027 mddev->queue->backing_dev_info.congested_fn = NULL; 5028 blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/ 5029 sysfs_remove_group(&mddev->kobj, &raid5_attrs_group); 5030 free_conf(conf); 5031 mddev->private = NULL; 5032 return 0; 5033} 5034 5035#ifdef DEBUG 5036static void print_sh(struct seq_file *seq, struct stripe_head *sh) 5037{ 5038 int i; 5039 5040 seq_printf(seq, "sh %llu, pd_idx %d, state %ld.\n", 5041 (unsigned long long)sh->sector, sh->pd_idx, sh->state); 5042 seq_printf(seq, "sh %llu, count %d.\n", 5043 (unsigned long long)sh->sector, atomic_read(&sh->count)); 5044 seq_printf(seq, "sh %llu, ", (unsigned long long)sh->sector); 5045 for (i = 0; i < sh->disks; i++) { 5046 seq_printf(seq, "(cache%d: %p %ld) ", 5047 i, sh->dev[i].page, sh->dev[i].flags); 5048 } 5049 seq_printf(seq, "\n"); 5050} 5051 5052static void printall(struct seq_file *seq, raid5_conf_t *conf) 5053{ 5054 struct stripe_head *sh; 5055 struct hlist_node *hn; 5056 int i; 5057 5058 spin_lock_irq(&conf->device_lock); 5059 for (i = 0; i < NR_HASH; i++) { 5060 hlist_for_each_entry(sh, hn, &conf->stripe_hashtbl[i], hash) { 5061 if (sh->raid_conf != conf) 5062 continue; 5063 print_sh(seq, sh); 5064 } 5065 } 5066 spin_unlock_irq(&conf->device_lock); 5067} 5068#endif 5069 5070static void status(struct seq_file *seq, mddev_t *mddev) 5071{ 5072 raid5_conf_t *conf = (raid5_conf_t *) mddev->private; 5073 int i; 5074 5075 seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level, 5076 mddev->chunk_sectors / 2, mddev->layout); 5077 seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded); 5078 for (i = 0; i < conf->raid_disks; i++) 5079 seq_printf (seq, "%s", 5080 conf->disks[i].rdev && 5081 test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_"); 5082 seq_printf (seq, "]"); 5083#ifdef DEBUG 5084 seq_printf (seq, "\n"); 5085 printall(seq, conf); 5086#endif 5087} 5088 5089static void print_raid5_conf (raid5_conf_t *conf) 5090{ 5091 int i; 5092 struct disk_info *tmp; 5093 5094 printk("RAID5 conf printout:\n"); 5095 if (!conf) { 5096 printk("(conf==NULL)\n"); 5097 return; 5098 } 5099 printk(" --- rd:%d wd:%d\n", conf->raid_disks, 5100 conf->raid_disks - conf->mddev->degraded); 5101 5102 for (i = 0; i < conf->raid_disks; i++) { 5103 char b[BDEVNAME_SIZE]; 5104 tmp = conf->disks + i; 5105 if (tmp->rdev) 5106 printk(" disk %d, o:%d, dev:%s\n", 5107 i, !test_bit(Faulty, &tmp->rdev->flags), 5108 bdevname(tmp->rdev->bdev,b)); 5109 } 5110} 5111 5112static int raid5_spare_active(mddev_t *mddev) 5113{ 5114 int i; 5115 raid5_conf_t *conf = mddev->private; 5116 struct disk_info *tmp; 5117 5118 for (i = 0; i < conf->raid_disks; i++) { 5119 tmp = conf->disks + i; 5120 if (tmp->rdev 5121 && !test_bit(Faulty, &tmp->rdev->flags) 5122 && !test_and_set_bit(In_sync, &tmp->rdev->flags)) { 5123 unsigned long flags; 5124 spin_lock_irqsave(&conf->device_lock, flags); 5125 mddev->degraded--; 5126 spin_unlock_irqrestore(&conf->device_lock, flags); 5127 } 5128 } 5129 print_raid5_conf(conf); 5130 return 0; 5131} 5132 5133static int raid5_remove_disk(mddev_t *mddev, int number) 5134{ 5135 raid5_conf_t *conf = mddev->private; 5136 int err = 0; 5137 mdk_rdev_t *rdev; 5138 struct disk_info *p = conf->disks + number; 5139 5140 print_raid5_conf(conf); 5141 rdev = p->rdev; 5142 if (rdev) { 5143 if (number >= conf->raid_disks && 5144 conf->reshape_progress == MaxSector) 5145 clear_bit(In_sync, &rdev->flags); 5146 5147 if (test_bit(In_sync, &rdev->flags) || 5148 atomic_read(&rdev->nr_pending)) { 5149 err = -EBUSY; 5150 goto abort; 5151 } 5152 /* Only remove non-faulty devices if recovery 5153 * isn't possible. 5154 */ 5155 if (!test_bit(Faulty, &rdev->flags) && 5156 mddev->degraded <= conf->max_degraded && 5157 number < conf->raid_disks) { 5158 err = -EBUSY; 5159 goto abort; 5160 } 5161 p->rdev = NULL; 5162 synchronize_rcu(); 5163 if (atomic_read(&rdev->nr_pending)) { 5164 /* lost the race, try later */ 5165 err = -EBUSY; 5166 p->rdev = rdev; 5167 } 5168 } 5169abort: 5170 5171 print_raid5_conf(conf); 5172 return err; 5173} 5174 5175static int raid5_add_disk(mddev_t *mddev, mdk_rdev_t *rdev) 5176{ 5177 raid5_conf_t *conf = mddev->private; 5178 int err = -EEXIST; 5179 int disk; 5180 struct disk_info *p; 5181 int first = 0; 5182 int last = conf->raid_disks - 1; 5183 5184 if (mddev->degraded > conf->max_degraded) 5185 /* no point adding a device */ 5186 return -EINVAL; 5187 5188 if (rdev->raid_disk >= 0) 5189 first = last = rdev->raid_disk; 5190 5191 /* 5192 * find the disk ... but prefer rdev->saved_raid_disk 5193 * if possible. 5194 */ 5195 if (rdev->saved_raid_disk >= 0 && 5196 rdev->saved_raid_disk >= first && 5197 conf->disks[rdev->saved_raid_disk].rdev == NULL) 5198 disk = rdev->saved_raid_disk; 5199 else 5200 disk = first; 5201 for ( ; disk <= last ; disk++) 5202 if ((p=conf->disks + disk)->rdev == NULL) { 5203 clear_bit(In_sync, &rdev->flags); 5204 rdev->raid_disk = disk; 5205 err = 0; 5206 if (rdev->saved_raid_disk != disk) 5207 conf->fullsync = 1; 5208 rcu_assign_pointer(p->rdev, rdev); 5209 break; 5210 } 5211 print_raid5_conf(conf); 5212 return err; 5213} 5214 5215static int raid5_resize(mddev_t *mddev, sector_t sectors) 5216{ 5217 /* no resync is happening, and there is enough space 5218 * on all devices, so we can resize. 5219 * We need to make sure resync covers any new space. 5220 * If the array is shrinking we should possibly wait until 5221 * any io in the removed space completes, but it hardly seems 5222 * worth it. 5223 */ 5224 sectors &= ~((sector_t)mddev->chunk_sectors - 1); 5225 md_set_array_sectors(mddev, raid5_size(mddev, sectors, 5226 mddev->raid_disks)); 5227 if (mddev->array_sectors > 5228 raid5_size(mddev, sectors, mddev->raid_disks)) 5229 return -EINVAL; 5230 set_capacity(mddev->gendisk, mddev->array_sectors); 5231 mddev->changed = 1; 5232 revalidate_disk(mddev->gendisk); 5233 if (sectors > mddev->dev_sectors && mddev->recovery_cp == MaxSector) { 5234 mddev->recovery_cp = mddev->dev_sectors; 5235 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 5236 } 5237 mddev->dev_sectors = sectors; 5238 mddev->resync_max_sectors = sectors; 5239 return 0; 5240} 5241 5242static int check_stripe_cache(mddev_t *mddev) 5243{ 5244 /* Can only proceed if there are plenty of stripe_heads. 5245 * We need a minimum of one full stripe,, and for sensible progress 5246 * it is best to have about 4 times that. 5247 * If we require 4 times, then the default 256 4K stripe_heads will 5248 * allow for chunk sizes up to 256K, which is probably OK. 5249 * If the chunk size is greater, user-space should request more 5250 * stripe_heads first. 5251 */ 5252 raid5_conf_t *conf = mddev->private; 5253 if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4 5254 > conf->max_nr_stripes || 5255 ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4 5256 > conf->max_nr_stripes) { 5257 printk(KERN_WARNING "raid5: reshape: not enough stripes. Needed %lu\n", 5258 ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9) 5259 / STRIPE_SIZE)*4); 5260 return 0; 5261 } 5262 return 1; 5263} 5264 5265static int check_reshape(mddev_t *mddev) 5266{ 5267 raid5_conf_t *conf = mddev->private; 5268 5269 if (mddev->delta_disks == 0 && 5270 mddev->new_layout == mddev->layout && 5271 mddev->new_chunk_sectors == mddev->chunk_sectors) 5272 return 0; /* nothing to do */ 5273 if (mddev->bitmap) 5274 /* Cannot grow a bitmap yet */ 5275 return -EBUSY; 5276 if (mddev->degraded > conf->max_degraded) 5277 return -EINVAL; 5278 if (mddev->delta_disks < 0) { 5279 /* We might be able to shrink, but the devices must 5280 * be made bigger first. 5281 * For raid6, 4 is the minimum size. 5282 * Otherwise 2 is the minimum 5283 */ 5284 int min = 2; 5285 if (mddev->level == 6) 5286 min = 4; 5287 if (mddev->raid_disks + mddev->delta_disks < min) 5288 return -EINVAL; 5289 } 5290 5291 if (!check_stripe_cache(mddev)) 5292 return -ENOSPC; 5293 5294 return resize_stripes(conf, conf->raid_disks + mddev->delta_disks); 5295} 5296 5297static int raid5_start_reshape(mddev_t *mddev) 5298{ 5299 raid5_conf_t *conf = mddev->private; 5300 mdk_rdev_t *rdev; 5301 int spares = 0; 5302 int added_devices = 0; 5303 unsigned long flags; 5304 5305 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) 5306 return -EBUSY; 5307 5308 if (!check_stripe_cache(mddev)) 5309 return -ENOSPC; 5310 5311 list_for_each_entry(rdev, &mddev->disks, same_set) 5312 if (rdev->raid_disk < 0 && 5313 !test_bit(Faulty, &rdev->flags)) 5314 spares++; 5315 5316 if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded) 5317 /* Not enough devices even to make a degraded array 5318 * of that size 5319 */ 5320 return -EINVAL; 5321 5322 /* Refuse to reduce size of the array. Any reductions in 5323 * array size must be through explicit setting of array_size 5324 * attribute. 5325 */ 5326 if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks) 5327 < mddev->array_sectors) { 5328 printk(KERN_ERR "md: %s: array size must be reduced " 5329 "before number of disks\n", mdname(mddev)); 5330 return -EINVAL; 5331 } 5332 5333 atomic_set(&conf->reshape_stripes, 0); 5334 spin_lock_irq(&conf->device_lock); 5335 conf->previous_raid_disks = conf->raid_disks; 5336 conf->raid_disks += mddev->delta_disks; 5337 conf->prev_chunk_sectors = conf->chunk_sectors; 5338 conf->chunk_sectors = mddev->new_chunk_sectors; 5339 conf->prev_algo = conf->algorithm; 5340 conf->algorithm = mddev->new_layout; 5341 if (mddev->delta_disks < 0) 5342 conf->reshape_progress = raid5_size(mddev, 0, 0); 5343 else 5344 conf->reshape_progress = 0; 5345 conf->reshape_safe = conf->reshape_progress; 5346 conf->generation++; 5347 spin_unlock_irq(&conf->device_lock); 5348 5349 /* Add some new drives, as many as will fit. 5350 * We know there are enough to make the newly sized array work. 5351 */ 5352 list_for_each_entry(rdev, &mddev->disks, same_set) 5353 if (rdev->raid_disk < 0 && 5354 !test_bit(Faulty, &rdev->flags)) { 5355 if (raid5_add_disk(mddev, rdev) == 0) { 5356 char nm[20]; 5357 set_bit(In_sync, &rdev->flags); 5358 added_devices++; 5359 rdev->recovery_offset = 0; 5360 sprintf(nm, "rd%d", rdev->raid_disk); 5361 if (sysfs_create_link(&mddev->kobj, 5362 &rdev->kobj, nm)) 5363 printk(KERN_WARNING 5364 "raid5: failed to create " 5365 " link %s for %s\n", 5366 nm, mdname(mddev)); 5367 } else 5368 break; 5369 } 5370 5371 if (mddev->delta_disks > 0) { 5372 spin_lock_irqsave(&conf->device_lock, flags); 5373 mddev->degraded = (conf->raid_disks - conf->previous_raid_disks) 5374 - added_devices; 5375 spin_unlock_irqrestore(&conf->device_lock, flags); 5376 } 5377 mddev->raid_disks = conf->raid_disks; 5378 mddev->reshape_position = conf->reshape_progress; 5379 set_bit(MD_CHANGE_DEVS, &mddev->flags); 5380 5381 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 5382 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 5383 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 5384 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 5385 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 5386 "reshape"); 5387 if (!mddev->sync_thread) { 5388 mddev->recovery = 0; 5389 spin_lock_irq(&conf->device_lock); 5390 mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks; 5391 conf->reshape_progress = MaxSector; 5392 spin_unlock_irq(&conf->device_lock); 5393 return -EAGAIN; 5394 } 5395 conf->reshape_checkpoint = jiffies; 5396 md_wakeup_thread(mddev->sync_thread); 5397 md_new_event(mddev); 5398 return 0; 5399} 5400 5401/* This is called from the reshape thread and should make any 5402 * changes needed in 'conf' 5403 */ 5404static void end_reshape(raid5_conf_t *conf) 5405{ 5406 5407 if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) { 5408 5409 spin_lock_irq(&conf->device_lock); 5410 conf->previous_raid_disks = conf->raid_disks; 5411 conf->reshape_progress = MaxSector; 5412 spin_unlock_irq(&conf->device_lock); 5413 wake_up(&conf->wait_for_overlap); 5414 5415 /* read-ahead size must cover two whole stripes, which is 5416 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices 5417 */ 5418 { 5419 int data_disks = conf->raid_disks - conf->max_degraded; 5420 int stripe = data_disks * ((conf->chunk_sectors << 9) 5421 / PAGE_SIZE); 5422 if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 5423 conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 5424 } 5425 } 5426} 5427 5428/* This is called from the raid5d thread with mddev_lock held. 5429 * It makes config changes to the device. 5430 */ 5431static void raid5_finish_reshape(mddev_t *mddev) 5432{ 5433 raid5_conf_t *conf = mddev->private; 5434 5435 if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) { 5436 5437 if (mddev->delta_disks > 0) { 5438 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); 5439 set_capacity(mddev->gendisk, mddev->array_sectors); 5440 mddev->changed = 1; 5441 revalidate_disk(mddev->gendisk); 5442 } else { 5443 int d; 5444 mddev->degraded = conf->raid_disks; 5445 for (d = 0; d < conf->raid_disks ; d++) 5446 if (conf->disks[d].rdev && 5447 test_bit(In_sync, 5448 &conf->disks[d].rdev->flags)) 5449 mddev->degraded--; 5450 for (d = conf->raid_disks ; 5451 d < conf->raid_disks - mddev->delta_disks; 5452 d++) { 5453 mdk_rdev_t *rdev = conf->disks[d].rdev; 5454 if (rdev && raid5_remove_disk(mddev, d) == 0) { 5455 char nm[20]; 5456 sprintf(nm, "rd%d", rdev->raid_disk); 5457 sysfs_remove_link(&mddev->kobj, nm); 5458 rdev->raid_disk = -1; 5459 } 5460 } 5461 } 5462 mddev->layout = conf->algorithm; 5463 mddev->chunk_sectors = conf->chunk_sectors; 5464 mddev->reshape_position = MaxSector; 5465 mddev->delta_disks = 0; 5466 } 5467} 5468 5469static void raid5_quiesce(mddev_t *mddev, int state) 5470{ 5471 raid5_conf_t *conf = mddev->private; 5472 5473 switch(state) { 5474 case 2: /* resume for a suspend */ 5475 wake_up(&conf->wait_for_overlap); 5476 break; 5477 5478 case 1: /* stop all writes */ 5479 spin_lock_irq(&conf->device_lock); 5480 /* '2' tells resync/reshape to pause so that all 5481 * active stripes can drain 5482 */ 5483 conf->quiesce = 2; 5484 wait_event_lock_irq(conf->wait_for_stripe, 5485 atomic_read(&conf->active_stripes) == 0 && 5486 atomic_read(&conf->active_aligned_reads) == 0, 5487 conf->device_lock, /* nothing */); 5488 conf->quiesce = 1; 5489 spin_unlock_irq(&conf->device_lock); 5490 /* allow reshape to continue */ 5491 wake_up(&conf->wait_for_overlap); 5492 break; 5493 5494 case 0: /* re-enable writes */ 5495 spin_lock_irq(&conf->device_lock); 5496 conf->quiesce = 0; 5497 wake_up(&conf->wait_for_stripe); 5498 wake_up(&conf->wait_for_overlap); 5499 spin_unlock_irq(&conf->device_lock); 5500 break; 5501 } 5502} 5503 5504 5505static void *raid5_takeover_raid1(mddev_t *mddev) 5506{ 5507 int chunksect; 5508 5509 if (mddev->raid_disks != 2 || 5510 mddev->degraded > 1) 5511 return ERR_PTR(-EINVAL); 5512 5513 /* Should check if there are write-behind devices? */ 5514 5515 chunksect = 64*2; /* 64K by default */ 5516 5517 /* The array must be an exact multiple of chunksize */ 5518 while (chunksect && (mddev->array_sectors & (chunksect-1))) 5519 chunksect >>= 1; 5520 5521 if ((chunksect<<9) < STRIPE_SIZE) 5522 /* array size does not allow a suitable chunk size */ 5523 return ERR_PTR(-EINVAL); 5524 5525 mddev->new_level = 5; 5526 mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC; 5527 mddev->new_chunk_sectors = chunksect; 5528 5529 return setup_conf(mddev); 5530} 5531 5532static void *raid5_takeover_raid6(mddev_t *mddev) 5533{ 5534 int new_layout; 5535 5536 switch (mddev->layout) { 5537 case ALGORITHM_LEFT_ASYMMETRIC_6: 5538 new_layout = ALGORITHM_LEFT_ASYMMETRIC; 5539 break; 5540 case ALGORITHM_RIGHT_ASYMMETRIC_6: 5541 new_layout = ALGORITHM_RIGHT_ASYMMETRIC; 5542 break; 5543 case ALGORITHM_LEFT_SYMMETRIC_6: 5544 new_layout = ALGORITHM_LEFT_SYMMETRIC; 5545 break; 5546 case ALGORITHM_RIGHT_SYMMETRIC_6: 5547 new_layout = ALGORITHM_RIGHT_SYMMETRIC; 5548 break; 5549 case ALGORITHM_PARITY_0_6: 5550 new_layout = ALGORITHM_PARITY_0; 5551 break; 5552 case ALGORITHM_PARITY_N: 5553 new_layout = ALGORITHM_PARITY_N; 5554 break; 5555 default: 5556 return ERR_PTR(-EINVAL); 5557 } 5558 mddev->new_level = 5; 5559 mddev->new_layout = new_layout; 5560 mddev->delta_disks = -1; 5561 mddev->raid_disks -= 1; 5562 return setup_conf(mddev); 5563} 5564 5565 5566static int raid5_check_reshape(mddev_t *mddev) 5567{ 5568 /* For a 2-drive array, the layout and chunk size can be changed 5569 * immediately as not restriping is needed. 5570 * For larger arrays we record the new value - after validation 5571 * to be used by a reshape pass. 5572 */ 5573 raid5_conf_t *conf = mddev->private; 5574 int new_chunk = mddev->new_chunk_sectors; 5575 5576 if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout)) 5577 return -EINVAL; 5578 if (new_chunk > 0) { 5579 if (!is_power_of_2(new_chunk)) 5580 return -EINVAL; 5581 if (new_chunk < (PAGE_SIZE>>9)) 5582 return -EINVAL; 5583 if (mddev->array_sectors & (new_chunk-1)) 5584 /* not factor of array size */ 5585 return -EINVAL; 5586 } 5587 5588 /* They look valid */ 5589 5590 if (mddev->raid_disks == 2) { 5591 /* can make the change immediately */ 5592 if (mddev->new_layout >= 0) { 5593 conf->algorithm = mddev->new_layout; 5594 mddev->layout = mddev->new_layout; 5595 } 5596 if (new_chunk > 0) { 5597 conf->chunk_sectors = new_chunk ; 5598 mddev->chunk_sectors = new_chunk; 5599 } 5600 set_bit(MD_CHANGE_DEVS, &mddev->flags); 5601 md_wakeup_thread(mddev->thread); 5602 } 5603 return check_reshape(mddev); 5604} 5605 5606static int raid6_check_reshape(mddev_t *mddev) 5607{ 5608 int new_chunk = mddev->new_chunk_sectors; 5609 5610 if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout)) 5611 return -EINVAL; 5612 if (new_chunk > 0) { 5613 if (!is_power_of_2(new_chunk)) 5614 return -EINVAL; 5615 if (new_chunk < (PAGE_SIZE >> 9)) 5616 return -EINVAL; 5617 if (mddev->array_sectors & (new_chunk-1)) 5618 /* not factor of array size */ 5619 return -EINVAL; 5620 } 5621 5622 /* They look valid */ 5623 return check_reshape(mddev); 5624} 5625 5626static void *raid5_takeover(mddev_t *mddev) 5627{ 5628 /* raid5 can take over: 5629 * raid0 - if all devices are the same - make it a raid4 layout 5630 * raid1 - if there are two drives. We need to know the chunk size 5631 * raid4 - trivial - just use a raid4 layout. 5632 * raid6 - Providing it is a *_6 layout 5633 */ 5634 5635 if (mddev->level == 1) 5636 return raid5_takeover_raid1(mddev); 5637 if (mddev->level == 4) { 5638 mddev->new_layout = ALGORITHM_PARITY_N; 5639 mddev->new_level = 5; 5640 return setup_conf(mddev); 5641 } 5642 if (mddev->level == 6) 5643 return raid5_takeover_raid6(mddev); 5644 5645 return ERR_PTR(-EINVAL); 5646} 5647 5648 5649static struct mdk_personality raid5_personality; 5650 5651static void *raid6_takeover(mddev_t *mddev) 5652{ 5653 /* Currently can only take over a raid5. We map the 5654 * personality to an equivalent raid6 personality 5655 * with the Q block at the end. 5656 */ 5657 int new_layout; 5658 5659 if (mddev->pers != &raid5_personality) 5660 return ERR_PTR(-EINVAL); 5661 if (mddev->degraded > 1) 5662 return ERR_PTR(-EINVAL); 5663 if (mddev->raid_disks > 253) 5664 return ERR_PTR(-EINVAL); 5665 if (mddev->raid_disks < 3) 5666 return ERR_PTR(-EINVAL); 5667 5668 switch (mddev->layout) { 5669 case ALGORITHM_LEFT_ASYMMETRIC: 5670 new_layout = ALGORITHM_LEFT_ASYMMETRIC_6; 5671 break; 5672 case ALGORITHM_RIGHT_ASYMMETRIC: 5673 new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6; 5674 break; 5675 case ALGORITHM_LEFT_SYMMETRIC: 5676 new_layout = ALGORITHM_LEFT_SYMMETRIC_6; 5677 break; 5678 case ALGORITHM_RIGHT_SYMMETRIC: 5679 new_layout = ALGORITHM_RIGHT_SYMMETRIC_6; 5680 break; 5681 case ALGORITHM_PARITY_0: 5682 new_layout = ALGORITHM_PARITY_0_6; 5683 break; 5684 case ALGORITHM_PARITY_N: 5685 new_layout = ALGORITHM_PARITY_N; 5686 break; 5687 default: 5688 return ERR_PTR(-EINVAL); 5689 } 5690 mddev->new_level = 6; 5691 mddev->new_layout = new_layout; 5692 mddev->delta_disks = 1; 5693 mddev->raid_disks += 1; 5694 return setup_conf(mddev); 5695} 5696 5697 5698static struct mdk_personality raid6_personality = 5699{ 5700 .name = "raid6", 5701 .level = 6, 5702 .owner = THIS_MODULE, 5703 .make_request = make_request, 5704 .run = run, 5705 .stop = stop, 5706 .status = status, 5707 .error_handler = error, 5708 .hot_add_disk = raid5_add_disk, 5709 .hot_remove_disk= raid5_remove_disk, 5710 .spare_active = raid5_spare_active, 5711 .sync_request = sync_request, 5712 .resize = raid5_resize, 5713 .size = raid5_size, 5714 .check_reshape = raid6_check_reshape, 5715 .start_reshape = raid5_start_reshape, 5716 .finish_reshape = raid5_finish_reshape, 5717 .quiesce = raid5_quiesce, 5718 .takeover = raid6_takeover, 5719}; 5720static struct mdk_personality raid5_personality = 5721{ 5722 .name = "raid5", 5723 .level = 5, 5724 .owner = THIS_MODULE, 5725 .make_request = make_request, 5726 .run = run, 5727 .stop = stop, 5728 .status = status, 5729 .error_handler = error, 5730 .hot_add_disk = raid5_add_disk, 5731 .hot_remove_disk= raid5_remove_disk, 5732 .spare_active = raid5_spare_active, 5733 .sync_request = sync_request, 5734 .resize = raid5_resize, 5735 .size = raid5_size, 5736 .check_reshape = raid5_check_reshape, 5737 .start_reshape = raid5_start_reshape, 5738 .finish_reshape = raid5_finish_reshape, 5739 .quiesce = raid5_quiesce, 5740 .takeover = raid5_takeover, 5741}; 5742 5743static struct mdk_personality raid4_personality = 5744{ 5745 .name = "raid4", 5746 .level = 4, 5747 .owner = THIS_MODULE, 5748 .make_request = make_request, 5749 .run = run, 5750 .stop = stop, 5751 .status = status, 5752 .error_handler = error, 5753 .hot_add_disk = raid5_add_disk, 5754 .hot_remove_disk= raid5_remove_disk, 5755 .spare_active = raid5_spare_active, 5756 .sync_request = sync_request, 5757 .resize = raid5_resize, 5758 .size = raid5_size, 5759 .check_reshape = raid5_check_reshape, 5760 .start_reshape = raid5_start_reshape, 5761 .finish_reshape = raid5_finish_reshape, 5762 .quiesce = raid5_quiesce, 5763}; 5764 5765static int __init raid5_init(void) 5766{ 5767 register_md_personality(&raid6_personality); 5768 register_md_personality(&raid5_personality); 5769 register_md_personality(&raid4_personality); 5770 return 0; 5771} 5772 5773static void raid5_exit(void) 5774{ 5775 unregister_md_personality(&raid6_personality); 5776 unregister_md_personality(&raid5_personality); 5777 unregister_md_personality(&raid4_personality); 5778} 5779 5780module_init(raid5_init); 5781module_exit(raid5_exit); 5782MODULE_LICENSE("GPL"); 5783MODULE_ALIAS("md-personality-4"); /* RAID5 */ 5784MODULE_ALIAS("md-raid5"); 5785MODULE_ALIAS("md-raid4"); 5786MODULE_ALIAS("md-level-5"); 5787MODULE_ALIAS("md-level-4"); 5788MODULE_ALIAS("md-personality-8"); /* RAID6 */ 5789MODULE_ALIAS("md-raid6"); 5790MODULE_ALIAS("md-level-6"); 5791 5792/* This used to be two separate modules, they were: */ 5793MODULE_ALIAS("raid5"); 5794MODULE_ALIAS("raid6"); 5795