backing-dev.c revision 0e175a1835ffc979e55787774e58ec79e41957d7
1 2#include <linux/wait.h> 3#include <linux/backing-dev.h> 4#include <linux/kthread.h> 5#include <linux/freezer.h> 6#include <linux/fs.h> 7#include <linux/pagemap.h> 8#include <linux/mm.h> 9#include <linux/sched.h> 10#include <linux/module.h> 11#include <linux/writeback.h> 12#include <linux/device.h> 13#include <trace/events/writeback.h> 14 15static atomic_long_t bdi_seq = ATOMIC_LONG_INIT(0); 16 17struct backing_dev_info default_backing_dev_info = { 18 .name = "default", 19 .ra_pages = VM_MAX_READAHEAD * 1024 / PAGE_CACHE_SIZE, 20 .state = 0, 21 .capabilities = BDI_CAP_MAP_COPY, 22}; 23EXPORT_SYMBOL_GPL(default_backing_dev_info); 24 25struct backing_dev_info noop_backing_dev_info = { 26 .name = "noop", 27 .capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK, 28}; 29EXPORT_SYMBOL_GPL(noop_backing_dev_info); 30 31static struct class *bdi_class; 32 33/* 34 * bdi_lock protects updates to bdi_list and bdi_pending_list, as well as 35 * reader side protection for bdi_pending_list. bdi_list has RCU reader side 36 * locking. 37 */ 38DEFINE_SPINLOCK(bdi_lock); 39LIST_HEAD(bdi_list); 40LIST_HEAD(bdi_pending_list); 41 42static struct task_struct *sync_supers_tsk; 43static struct timer_list sync_supers_timer; 44 45static int bdi_sync_supers(void *); 46static void sync_supers_timer_fn(unsigned long); 47 48void bdi_lock_two(struct bdi_writeback *wb1, struct bdi_writeback *wb2) 49{ 50 if (wb1 < wb2) { 51 spin_lock(&wb1->list_lock); 52 spin_lock_nested(&wb2->list_lock, 1); 53 } else { 54 spin_lock(&wb2->list_lock); 55 spin_lock_nested(&wb1->list_lock, 1); 56 } 57} 58 59#ifdef CONFIG_DEBUG_FS 60#include <linux/debugfs.h> 61#include <linux/seq_file.h> 62 63static struct dentry *bdi_debug_root; 64 65static void bdi_debug_init(void) 66{ 67 bdi_debug_root = debugfs_create_dir("bdi", NULL); 68} 69 70static int bdi_debug_stats_show(struct seq_file *m, void *v) 71{ 72 struct backing_dev_info *bdi = m->private; 73 struct bdi_writeback *wb = &bdi->wb; 74 unsigned long background_thresh; 75 unsigned long dirty_thresh; 76 unsigned long bdi_thresh; 77 unsigned long nr_dirty, nr_io, nr_more_io; 78 struct inode *inode; 79 80 nr_dirty = nr_io = nr_more_io = 0; 81 spin_lock(&wb->list_lock); 82 list_for_each_entry(inode, &wb->b_dirty, i_wb_list) 83 nr_dirty++; 84 list_for_each_entry(inode, &wb->b_io, i_wb_list) 85 nr_io++; 86 list_for_each_entry(inode, &wb->b_more_io, i_wb_list) 87 nr_more_io++; 88 spin_unlock(&wb->list_lock); 89 90 global_dirty_limits(&background_thresh, &dirty_thresh); 91 bdi_thresh = bdi_dirty_limit(bdi, dirty_thresh); 92 93#define K(x) ((x) << (PAGE_SHIFT - 10)) 94 seq_printf(m, 95 "BdiWriteback: %10lu kB\n" 96 "BdiReclaimable: %10lu kB\n" 97 "BdiDirtyThresh: %10lu kB\n" 98 "DirtyThresh: %10lu kB\n" 99 "BackgroundThresh: %10lu kB\n" 100 "BdiDirtied: %10lu kB\n" 101 "BdiWritten: %10lu kB\n" 102 "BdiWriteBandwidth: %10lu kBps\n" 103 "b_dirty: %10lu\n" 104 "b_io: %10lu\n" 105 "b_more_io: %10lu\n" 106 "bdi_list: %10u\n" 107 "state: %10lx\n", 108 (unsigned long) K(bdi_stat(bdi, BDI_WRITEBACK)), 109 (unsigned long) K(bdi_stat(bdi, BDI_RECLAIMABLE)), 110 K(bdi_thresh), 111 K(dirty_thresh), 112 K(background_thresh), 113 (unsigned long) K(bdi_stat(bdi, BDI_DIRTIED)), 114 (unsigned long) K(bdi_stat(bdi, BDI_WRITTEN)), 115 (unsigned long) K(bdi->write_bandwidth), 116 nr_dirty, 117 nr_io, 118 nr_more_io, 119 !list_empty(&bdi->bdi_list), bdi->state); 120#undef K 121 122 return 0; 123} 124 125static int bdi_debug_stats_open(struct inode *inode, struct file *file) 126{ 127 return single_open(file, bdi_debug_stats_show, inode->i_private); 128} 129 130static const struct file_operations bdi_debug_stats_fops = { 131 .open = bdi_debug_stats_open, 132 .read = seq_read, 133 .llseek = seq_lseek, 134 .release = single_release, 135}; 136 137static void bdi_debug_register(struct backing_dev_info *bdi, const char *name) 138{ 139 bdi->debug_dir = debugfs_create_dir(name, bdi_debug_root); 140 bdi->debug_stats = debugfs_create_file("stats", 0444, bdi->debug_dir, 141 bdi, &bdi_debug_stats_fops); 142} 143 144static void bdi_debug_unregister(struct backing_dev_info *bdi) 145{ 146 debugfs_remove(bdi->debug_stats); 147 debugfs_remove(bdi->debug_dir); 148} 149#else 150static inline void bdi_debug_init(void) 151{ 152} 153static inline void bdi_debug_register(struct backing_dev_info *bdi, 154 const char *name) 155{ 156} 157static inline void bdi_debug_unregister(struct backing_dev_info *bdi) 158{ 159} 160#endif 161 162static ssize_t read_ahead_kb_store(struct device *dev, 163 struct device_attribute *attr, 164 const char *buf, size_t count) 165{ 166 struct backing_dev_info *bdi = dev_get_drvdata(dev); 167 char *end; 168 unsigned long read_ahead_kb; 169 ssize_t ret = -EINVAL; 170 171 read_ahead_kb = simple_strtoul(buf, &end, 10); 172 if (*buf && (end[0] == '\0' || (end[0] == '\n' && end[1] == '\0'))) { 173 bdi->ra_pages = read_ahead_kb >> (PAGE_SHIFT - 10); 174 ret = count; 175 } 176 return ret; 177} 178 179#define K(pages) ((pages) << (PAGE_SHIFT - 10)) 180 181#define BDI_SHOW(name, expr) \ 182static ssize_t name##_show(struct device *dev, \ 183 struct device_attribute *attr, char *page) \ 184{ \ 185 struct backing_dev_info *bdi = dev_get_drvdata(dev); \ 186 \ 187 return snprintf(page, PAGE_SIZE-1, "%lld\n", (long long)expr); \ 188} 189 190BDI_SHOW(read_ahead_kb, K(bdi->ra_pages)) 191 192static ssize_t min_ratio_store(struct device *dev, 193 struct device_attribute *attr, const char *buf, size_t count) 194{ 195 struct backing_dev_info *bdi = dev_get_drvdata(dev); 196 char *end; 197 unsigned int ratio; 198 ssize_t ret = -EINVAL; 199 200 ratio = simple_strtoul(buf, &end, 10); 201 if (*buf && (end[0] == '\0' || (end[0] == '\n' && end[1] == '\0'))) { 202 ret = bdi_set_min_ratio(bdi, ratio); 203 if (!ret) 204 ret = count; 205 } 206 return ret; 207} 208BDI_SHOW(min_ratio, bdi->min_ratio) 209 210static ssize_t max_ratio_store(struct device *dev, 211 struct device_attribute *attr, const char *buf, size_t count) 212{ 213 struct backing_dev_info *bdi = dev_get_drvdata(dev); 214 char *end; 215 unsigned int ratio; 216 ssize_t ret = -EINVAL; 217 218 ratio = simple_strtoul(buf, &end, 10); 219 if (*buf && (end[0] == '\0' || (end[0] == '\n' && end[1] == '\0'))) { 220 ret = bdi_set_max_ratio(bdi, ratio); 221 if (!ret) 222 ret = count; 223 } 224 return ret; 225} 226BDI_SHOW(max_ratio, bdi->max_ratio) 227 228#define __ATTR_RW(attr) __ATTR(attr, 0644, attr##_show, attr##_store) 229 230static struct device_attribute bdi_dev_attrs[] = { 231 __ATTR_RW(read_ahead_kb), 232 __ATTR_RW(min_ratio), 233 __ATTR_RW(max_ratio), 234 __ATTR_NULL, 235}; 236 237static __init int bdi_class_init(void) 238{ 239 bdi_class = class_create(THIS_MODULE, "bdi"); 240 if (IS_ERR(bdi_class)) 241 return PTR_ERR(bdi_class); 242 243 bdi_class->dev_attrs = bdi_dev_attrs; 244 bdi_debug_init(); 245 return 0; 246} 247postcore_initcall(bdi_class_init); 248 249static int __init default_bdi_init(void) 250{ 251 int err; 252 253 sync_supers_tsk = kthread_run(bdi_sync_supers, NULL, "sync_supers"); 254 BUG_ON(IS_ERR(sync_supers_tsk)); 255 256 setup_timer(&sync_supers_timer, sync_supers_timer_fn, 0); 257 bdi_arm_supers_timer(); 258 259 err = bdi_init(&default_backing_dev_info); 260 if (!err) 261 bdi_register(&default_backing_dev_info, NULL, "default"); 262 err = bdi_init(&noop_backing_dev_info); 263 264 return err; 265} 266subsys_initcall(default_bdi_init); 267 268int bdi_has_dirty_io(struct backing_dev_info *bdi) 269{ 270 return wb_has_dirty_io(&bdi->wb); 271} 272 273/* 274 * kupdated() used to do this. We cannot do it from the bdi_forker_thread() 275 * or we risk deadlocking on ->s_umount. The longer term solution would be 276 * to implement sync_supers_bdi() or similar and simply do it from the 277 * bdi writeback thread individually. 278 */ 279static int bdi_sync_supers(void *unused) 280{ 281 set_user_nice(current, 0); 282 283 while (!kthread_should_stop()) { 284 set_current_state(TASK_INTERRUPTIBLE); 285 schedule(); 286 287 /* 288 * Do this periodically, like kupdated() did before. 289 */ 290 sync_supers(); 291 } 292 293 return 0; 294} 295 296void bdi_arm_supers_timer(void) 297{ 298 unsigned long next; 299 300 if (!dirty_writeback_interval) 301 return; 302 303 next = msecs_to_jiffies(dirty_writeback_interval * 10) + jiffies; 304 mod_timer(&sync_supers_timer, round_jiffies_up(next)); 305} 306 307static void sync_supers_timer_fn(unsigned long unused) 308{ 309 wake_up_process(sync_supers_tsk); 310 bdi_arm_supers_timer(); 311} 312 313static void wakeup_timer_fn(unsigned long data) 314{ 315 struct backing_dev_info *bdi = (struct backing_dev_info *)data; 316 317 spin_lock_bh(&bdi->wb_lock); 318 if (bdi->wb.task) { 319 trace_writeback_wake_thread(bdi); 320 wake_up_process(bdi->wb.task); 321 } else { 322 /* 323 * When bdi tasks are inactive for long time, they are killed. 324 * In this case we have to wake-up the forker thread which 325 * should create and run the bdi thread. 326 */ 327 trace_writeback_wake_forker_thread(bdi); 328 wake_up_process(default_backing_dev_info.wb.task); 329 } 330 spin_unlock_bh(&bdi->wb_lock); 331} 332 333/* 334 * This function is used when the first inode for this bdi is marked dirty. It 335 * wakes-up the corresponding bdi thread which should then take care of the 336 * periodic background write-out of dirty inodes. Since the write-out would 337 * starts only 'dirty_writeback_interval' centisecs from now anyway, we just 338 * set up a timer which wakes the bdi thread up later. 339 * 340 * Note, we wouldn't bother setting up the timer, but this function is on the 341 * fast-path (used by '__mark_inode_dirty()'), so we save few context switches 342 * by delaying the wake-up. 343 */ 344void bdi_wakeup_thread_delayed(struct backing_dev_info *bdi) 345{ 346 unsigned long timeout; 347 348 timeout = msecs_to_jiffies(dirty_writeback_interval * 10); 349 mod_timer(&bdi->wb.wakeup_timer, jiffies + timeout); 350} 351 352/* 353 * Calculate the longest interval (jiffies) bdi threads are allowed to be 354 * inactive. 355 */ 356static unsigned long bdi_longest_inactive(void) 357{ 358 unsigned long interval; 359 360 interval = msecs_to_jiffies(dirty_writeback_interval * 10); 361 return max(5UL * 60 * HZ, interval); 362} 363 364/* 365 * Clear pending bit and wakeup anybody waiting for flusher thread creation or 366 * shutdown 367 */ 368static void bdi_clear_pending(struct backing_dev_info *bdi) 369{ 370 clear_bit(BDI_pending, &bdi->state); 371 smp_mb__after_clear_bit(); 372 wake_up_bit(&bdi->state, BDI_pending); 373} 374 375static int bdi_forker_thread(void *ptr) 376{ 377 struct bdi_writeback *me = ptr; 378 379 current->flags |= PF_SWAPWRITE; 380 set_freezable(); 381 382 /* 383 * Our parent may run at a different priority, just set us to normal 384 */ 385 set_user_nice(current, 0); 386 387 for (;;) { 388 struct task_struct *task = NULL; 389 struct backing_dev_info *bdi; 390 enum { 391 NO_ACTION, /* Nothing to do */ 392 FORK_THREAD, /* Fork bdi thread */ 393 KILL_THREAD, /* Kill inactive bdi thread */ 394 } action = NO_ACTION; 395 396 /* 397 * Temporary measure, we want to make sure we don't see 398 * dirty data on the default backing_dev_info 399 */ 400 if (wb_has_dirty_io(me) || !list_empty(&me->bdi->work_list)) { 401 del_timer(&me->wakeup_timer); 402 wb_do_writeback(me, 0); 403 } 404 405 spin_lock_bh(&bdi_lock); 406 /* 407 * In the following loop we are going to check whether we have 408 * some work to do without any synchronization with tasks 409 * waking us up to do work for them. So we have to set task 410 * state already here so that we don't miss wakeups coming 411 * after we verify some condition. 412 */ 413 set_current_state(TASK_INTERRUPTIBLE); 414 415 list_for_each_entry(bdi, &bdi_list, bdi_list) { 416 bool have_dirty_io; 417 418 if (!bdi_cap_writeback_dirty(bdi) || 419 bdi_cap_flush_forker(bdi)) 420 continue; 421 422 WARN(!test_bit(BDI_registered, &bdi->state), 423 "bdi %p/%s is not registered!\n", bdi, bdi->name); 424 425 have_dirty_io = !list_empty(&bdi->work_list) || 426 wb_has_dirty_io(&bdi->wb); 427 428 /* 429 * If the bdi has work to do, but the thread does not 430 * exist - create it. 431 */ 432 if (!bdi->wb.task && have_dirty_io) { 433 /* 434 * Set the pending bit - if someone will try to 435 * unregister this bdi - it'll wait on this bit. 436 */ 437 set_bit(BDI_pending, &bdi->state); 438 action = FORK_THREAD; 439 break; 440 } 441 442 spin_lock(&bdi->wb_lock); 443 444 /* 445 * If there is no work to do and the bdi thread was 446 * inactive long enough - kill it. The wb_lock is taken 447 * to make sure no-one adds more work to this bdi and 448 * wakes the bdi thread up. 449 */ 450 if (bdi->wb.task && !have_dirty_io && 451 time_after(jiffies, bdi->wb.last_active + 452 bdi_longest_inactive())) { 453 task = bdi->wb.task; 454 bdi->wb.task = NULL; 455 spin_unlock(&bdi->wb_lock); 456 set_bit(BDI_pending, &bdi->state); 457 action = KILL_THREAD; 458 break; 459 } 460 spin_unlock(&bdi->wb_lock); 461 } 462 spin_unlock_bh(&bdi_lock); 463 464 /* Keep working if default bdi still has things to do */ 465 if (!list_empty(&me->bdi->work_list)) 466 __set_current_state(TASK_RUNNING); 467 468 switch (action) { 469 case FORK_THREAD: 470 __set_current_state(TASK_RUNNING); 471 task = kthread_create(bdi_writeback_thread, &bdi->wb, 472 "flush-%s", dev_name(bdi->dev)); 473 if (IS_ERR(task)) { 474 /* 475 * If thread creation fails, force writeout of 476 * the bdi from the thread. Hopefully 1024 is 477 * large enough for efficient IO. 478 */ 479 writeback_inodes_wb(&bdi->wb, 1024, 480 WB_REASON_FORKER_THREAD); 481 } else { 482 /* 483 * The spinlock makes sure we do not lose 484 * wake-ups when racing with 'bdi_queue_work()'. 485 * And as soon as the bdi thread is visible, we 486 * can start it. 487 */ 488 spin_lock_bh(&bdi->wb_lock); 489 bdi->wb.task = task; 490 spin_unlock_bh(&bdi->wb_lock); 491 wake_up_process(task); 492 } 493 bdi_clear_pending(bdi); 494 break; 495 496 case KILL_THREAD: 497 __set_current_state(TASK_RUNNING); 498 kthread_stop(task); 499 bdi_clear_pending(bdi); 500 break; 501 502 case NO_ACTION: 503 if (!wb_has_dirty_io(me) || !dirty_writeback_interval) 504 /* 505 * There are no dirty data. The only thing we 506 * should now care about is checking for 507 * inactive bdi threads and killing them. Thus, 508 * let's sleep for longer time, save energy and 509 * be friendly for battery-driven devices. 510 */ 511 schedule_timeout(bdi_longest_inactive()); 512 else 513 schedule_timeout(msecs_to_jiffies(dirty_writeback_interval * 10)); 514 try_to_freeze(); 515 break; 516 } 517 } 518 519 return 0; 520} 521 522/* 523 * Remove bdi from bdi_list, and ensure that it is no longer visible 524 */ 525static void bdi_remove_from_list(struct backing_dev_info *bdi) 526{ 527 spin_lock_bh(&bdi_lock); 528 list_del_rcu(&bdi->bdi_list); 529 spin_unlock_bh(&bdi_lock); 530 531 synchronize_rcu_expedited(); 532} 533 534int bdi_register(struct backing_dev_info *bdi, struct device *parent, 535 const char *fmt, ...) 536{ 537 va_list args; 538 struct device *dev; 539 540 if (bdi->dev) /* The driver needs to use separate queues per device */ 541 return 0; 542 543 va_start(args, fmt); 544 dev = device_create_vargs(bdi_class, parent, MKDEV(0, 0), bdi, fmt, args); 545 va_end(args); 546 if (IS_ERR(dev)) 547 return PTR_ERR(dev); 548 549 bdi->dev = dev; 550 551 /* 552 * Just start the forker thread for our default backing_dev_info, 553 * and add other bdi's to the list. They will get a thread created 554 * on-demand when they need it. 555 */ 556 if (bdi_cap_flush_forker(bdi)) { 557 struct bdi_writeback *wb = &bdi->wb; 558 559 wb->task = kthread_run(bdi_forker_thread, wb, "bdi-%s", 560 dev_name(dev)); 561 if (IS_ERR(wb->task)) 562 return PTR_ERR(wb->task); 563 } 564 565 bdi_debug_register(bdi, dev_name(dev)); 566 set_bit(BDI_registered, &bdi->state); 567 568 spin_lock_bh(&bdi_lock); 569 list_add_tail_rcu(&bdi->bdi_list, &bdi_list); 570 spin_unlock_bh(&bdi_lock); 571 572 trace_writeback_bdi_register(bdi); 573 return 0; 574} 575EXPORT_SYMBOL(bdi_register); 576 577int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev) 578{ 579 return bdi_register(bdi, NULL, "%u:%u", MAJOR(dev), MINOR(dev)); 580} 581EXPORT_SYMBOL(bdi_register_dev); 582 583/* 584 * Remove bdi from the global list and shutdown any threads we have running 585 */ 586static void bdi_wb_shutdown(struct backing_dev_info *bdi) 587{ 588 if (!bdi_cap_writeback_dirty(bdi)) 589 return; 590 591 /* 592 * Make sure nobody finds us on the bdi_list anymore 593 */ 594 bdi_remove_from_list(bdi); 595 596 /* 597 * If setup is pending, wait for that to complete first 598 */ 599 wait_on_bit(&bdi->state, BDI_pending, bdi_sched_wait, 600 TASK_UNINTERRUPTIBLE); 601 602 /* 603 * Finally, kill the kernel thread. We don't need to be RCU 604 * safe anymore, since the bdi is gone from visibility. Force 605 * unfreeze of the thread before calling kthread_stop(), otherwise 606 * it would never exet if it is currently stuck in the refrigerator. 607 */ 608 if (bdi->wb.task) { 609 thaw_process(bdi->wb.task); 610 kthread_stop(bdi->wb.task); 611 } 612} 613 614/* 615 * This bdi is going away now, make sure that no super_blocks point to it 616 */ 617static void bdi_prune_sb(struct backing_dev_info *bdi) 618{ 619 struct super_block *sb; 620 621 spin_lock(&sb_lock); 622 list_for_each_entry(sb, &super_blocks, s_list) { 623 if (sb->s_bdi == bdi) 624 sb->s_bdi = &default_backing_dev_info; 625 } 626 spin_unlock(&sb_lock); 627} 628 629void bdi_unregister(struct backing_dev_info *bdi) 630{ 631 if (bdi->dev) { 632 bdi_set_min_ratio(bdi, 0); 633 trace_writeback_bdi_unregister(bdi); 634 bdi_prune_sb(bdi); 635 del_timer_sync(&bdi->wb.wakeup_timer); 636 637 if (!bdi_cap_flush_forker(bdi)) 638 bdi_wb_shutdown(bdi); 639 bdi_debug_unregister(bdi); 640 device_unregister(bdi->dev); 641 bdi->dev = NULL; 642 } 643} 644EXPORT_SYMBOL(bdi_unregister); 645 646static void bdi_wb_init(struct bdi_writeback *wb, struct backing_dev_info *bdi) 647{ 648 memset(wb, 0, sizeof(*wb)); 649 650 wb->bdi = bdi; 651 wb->last_old_flush = jiffies; 652 INIT_LIST_HEAD(&wb->b_dirty); 653 INIT_LIST_HEAD(&wb->b_io); 654 INIT_LIST_HEAD(&wb->b_more_io); 655 spin_lock_init(&wb->list_lock); 656 setup_timer(&wb->wakeup_timer, wakeup_timer_fn, (unsigned long)bdi); 657} 658 659/* 660 * Initial write bandwidth: 100 MB/s 661 */ 662#define INIT_BW (100 << (20 - PAGE_SHIFT)) 663 664int bdi_init(struct backing_dev_info *bdi) 665{ 666 int i, err; 667 668 bdi->dev = NULL; 669 670 bdi->min_ratio = 0; 671 bdi->max_ratio = 100; 672 bdi->max_prop_frac = PROP_FRAC_BASE; 673 spin_lock_init(&bdi->wb_lock); 674 INIT_LIST_HEAD(&bdi->bdi_list); 675 INIT_LIST_HEAD(&bdi->work_list); 676 677 bdi_wb_init(&bdi->wb, bdi); 678 679 for (i = 0; i < NR_BDI_STAT_ITEMS; i++) { 680 err = percpu_counter_init(&bdi->bdi_stat[i], 0); 681 if (err) 682 goto err; 683 } 684 685 bdi->dirty_exceeded = 0; 686 687 bdi->bw_time_stamp = jiffies; 688 bdi->written_stamp = 0; 689 690 bdi->balanced_dirty_ratelimit = INIT_BW; 691 bdi->dirty_ratelimit = INIT_BW; 692 bdi->write_bandwidth = INIT_BW; 693 bdi->avg_write_bandwidth = INIT_BW; 694 695 err = prop_local_init_percpu(&bdi->completions); 696 697 if (err) { 698err: 699 while (i--) 700 percpu_counter_destroy(&bdi->bdi_stat[i]); 701 } 702 703 return err; 704} 705EXPORT_SYMBOL(bdi_init); 706 707void bdi_destroy(struct backing_dev_info *bdi) 708{ 709 int i; 710 711 /* 712 * Splice our entries to the default_backing_dev_info, if this 713 * bdi disappears 714 */ 715 if (bdi_has_dirty_io(bdi)) { 716 struct bdi_writeback *dst = &default_backing_dev_info.wb; 717 718 bdi_lock_two(&bdi->wb, dst); 719 list_splice(&bdi->wb.b_dirty, &dst->b_dirty); 720 list_splice(&bdi->wb.b_io, &dst->b_io); 721 list_splice(&bdi->wb.b_more_io, &dst->b_more_io); 722 spin_unlock(&bdi->wb.list_lock); 723 spin_unlock(&dst->list_lock); 724 } 725 726 bdi_unregister(bdi); 727 728 for (i = 0; i < NR_BDI_STAT_ITEMS; i++) 729 percpu_counter_destroy(&bdi->bdi_stat[i]); 730 731 prop_local_destroy_percpu(&bdi->completions); 732} 733EXPORT_SYMBOL(bdi_destroy); 734 735/* 736 * For use from filesystems to quickly init and register a bdi associated 737 * with dirty writeback 738 */ 739int bdi_setup_and_register(struct backing_dev_info *bdi, char *name, 740 unsigned int cap) 741{ 742 char tmp[32]; 743 int err; 744 745 bdi->name = name; 746 bdi->capabilities = cap; 747 err = bdi_init(bdi); 748 if (err) 749 return err; 750 751 sprintf(tmp, "%.28s%s", name, "-%d"); 752 err = bdi_register(bdi, NULL, tmp, atomic_long_inc_return(&bdi_seq)); 753 if (err) { 754 bdi_destroy(bdi); 755 return err; 756 } 757 758 return 0; 759} 760EXPORT_SYMBOL(bdi_setup_and_register); 761 762static wait_queue_head_t congestion_wqh[2] = { 763 __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[0]), 764 __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[1]) 765 }; 766static atomic_t nr_bdi_congested[2]; 767 768void clear_bdi_congested(struct backing_dev_info *bdi, int sync) 769{ 770 enum bdi_state bit; 771 wait_queue_head_t *wqh = &congestion_wqh[sync]; 772 773 bit = sync ? BDI_sync_congested : BDI_async_congested; 774 if (test_and_clear_bit(bit, &bdi->state)) 775 atomic_dec(&nr_bdi_congested[sync]); 776 smp_mb__after_clear_bit(); 777 if (waitqueue_active(wqh)) 778 wake_up(wqh); 779} 780EXPORT_SYMBOL(clear_bdi_congested); 781 782void set_bdi_congested(struct backing_dev_info *bdi, int sync) 783{ 784 enum bdi_state bit; 785 786 bit = sync ? BDI_sync_congested : BDI_async_congested; 787 if (!test_and_set_bit(bit, &bdi->state)) 788 atomic_inc(&nr_bdi_congested[sync]); 789} 790EXPORT_SYMBOL(set_bdi_congested); 791 792/** 793 * congestion_wait - wait for a backing_dev to become uncongested 794 * @sync: SYNC or ASYNC IO 795 * @timeout: timeout in jiffies 796 * 797 * Waits for up to @timeout jiffies for a backing_dev (any backing_dev) to exit 798 * write congestion. If no backing_devs are congested then just wait for the 799 * next write to be completed. 800 */ 801long congestion_wait(int sync, long timeout) 802{ 803 long ret; 804 unsigned long start = jiffies; 805 DEFINE_WAIT(wait); 806 wait_queue_head_t *wqh = &congestion_wqh[sync]; 807 808 prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE); 809 ret = io_schedule_timeout(timeout); 810 finish_wait(wqh, &wait); 811 812 trace_writeback_congestion_wait(jiffies_to_usecs(timeout), 813 jiffies_to_usecs(jiffies - start)); 814 815 return ret; 816} 817EXPORT_SYMBOL(congestion_wait); 818 819/** 820 * wait_iff_congested - Conditionally wait for a backing_dev to become uncongested or a zone to complete writes 821 * @zone: A zone to check if it is heavily congested 822 * @sync: SYNC or ASYNC IO 823 * @timeout: timeout in jiffies 824 * 825 * In the event of a congested backing_dev (any backing_dev) and the given 826 * @zone has experienced recent congestion, this waits for up to @timeout 827 * jiffies for either a BDI to exit congestion of the given @sync queue 828 * or a write to complete. 829 * 830 * In the absence of zone congestion, cond_resched() is called to yield 831 * the processor if necessary but otherwise does not sleep. 832 * 833 * The return value is 0 if the sleep is for the full timeout. Otherwise, 834 * it is the number of jiffies that were still remaining when the function 835 * returned. return_value == timeout implies the function did not sleep. 836 */ 837long wait_iff_congested(struct zone *zone, int sync, long timeout) 838{ 839 long ret; 840 unsigned long start = jiffies; 841 DEFINE_WAIT(wait); 842 wait_queue_head_t *wqh = &congestion_wqh[sync]; 843 844 /* 845 * If there is no congestion, or heavy congestion is not being 846 * encountered in the current zone, yield if necessary instead 847 * of sleeping on the congestion queue 848 */ 849 if (atomic_read(&nr_bdi_congested[sync]) == 0 || 850 !zone_is_reclaim_congested(zone)) { 851 cond_resched(); 852 853 /* In case we scheduled, work out time remaining */ 854 ret = timeout - (jiffies - start); 855 if (ret < 0) 856 ret = 0; 857 858 goto out; 859 } 860 861 /* Sleep until uncongested or a write happens */ 862 prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE); 863 ret = io_schedule_timeout(timeout); 864 finish_wait(wqh, &wait); 865 866out: 867 trace_writeback_wait_iff_congested(jiffies_to_usecs(timeout), 868 jiffies_to_usecs(jiffies - start)); 869 870 return ret; 871} 872EXPORT_SYMBOL(wait_iff_congested); 873