page-writeback.c revision 1fe06ad89255c211fe100d7f690d10b161398df8
1/* 2 * mm/page-writeback.c 3 * 4 * Copyright (C) 2002, Linus Torvalds. 5 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> 6 * 7 * Contains functions related to writing back dirty pages at the 8 * address_space level. 9 * 10 * 10Apr2002 Andrew Morton 11 * Initial version 12 */ 13 14#include <linux/kernel.h> 15#include <linux/module.h> 16#include <linux/spinlock.h> 17#include <linux/fs.h> 18#include <linux/mm.h> 19#include <linux/swap.h> 20#include <linux/slab.h> 21#include <linux/pagemap.h> 22#include <linux/writeback.h> 23#include <linux/init.h> 24#include <linux/backing-dev.h> 25#include <linux/task_io_accounting_ops.h> 26#include <linux/blkdev.h> 27#include <linux/mpage.h> 28#include <linux/rmap.h> 29#include <linux/percpu.h> 30#include <linux/notifier.h> 31#include <linux/smp.h> 32#include <linux/sysctl.h> 33#include <linux/cpu.h> 34#include <linux/syscalls.h> 35#include <linux/buffer_head.h> 36#include <linux/pagevec.h> 37 38/* 39 * After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited 40 * will look to see if it needs to force writeback or throttling. 41 */ 42static long ratelimit_pages = 32; 43 44/* 45 * When balance_dirty_pages decides that the caller needs to perform some 46 * non-background writeback, this is how many pages it will attempt to write. 47 * It should be somewhat larger than RATELIMIT_PAGES to ensure that reasonably 48 * large amounts of I/O are submitted. 49 */ 50static inline long sync_writeback_pages(void) 51{ 52 return ratelimit_pages + ratelimit_pages / 2; 53} 54 55/* The following parameters are exported via /proc/sys/vm */ 56 57/* 58 * Start background writeback (via pdflush) at this percentage 59 */ 60int dirty_background_ratio = 10; 61 62/* 63 * dirty_background_bytes starts at 0 (disabled) so that it is a function of 64 * dirty_background_ratio * the amount of dirtyable memory 65 */ 66unsigned long dirty_background_bytes; 67 68/* 69 * free highmem will not be subtracted from the total free memory 70 * for calculating free ratios if vm_highmem_is_dirtyable is true 71 */ 72int vm_highmem_is_dirtyable; 73 74/* 75 * The generator of dirty data starts writeback at this percentage 76 */ 77int vm_dirty_ratio = 20; 78 79/* 80 * vm_dirty_bytes starts at 0 (disabled) so that it is a function of 81 * vm_dirty_ratio * the amount of dirtyable memory 82 */ 83unsigned long vm_dirty_bytes; 84 85/* 86 * The interval between `kupdate'-style writebacks 87 */ 88unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */ 89 90/* 91 * The longest time for which data is allowed to remain dirty 92 */ 93unsigned int dirty_expire_interval = 30 * 100; /* centiseconds */ 94 95/* 96 * Flag that makes the machine dump writes/reads and block dirtyings. 97 */ 98int block_dump; 99 100/* 101 * Flag that puts the machine in "laptop mode". Doubles as a timeout in jiffies: 102 * a full sync is triggered after this time elapses without any disk activity. 103 */ 104int laptop_mode; 105 106EXPORT_SYMBOL(laptop_mode); 107 108/* End of sysctl-exported parameters */ 109 110 111/* 112 * Scale the writeback cache size proportional to the relative writeout speeds. 113 * 114 * We do this by keeping a floating proportion between BDIs, based on page 115 * writeback completions [end_page_writeback()]. Those devices that write out 116 * pages fastest will get the larger share, while the slower will get a smaller 117 * share. 118 * 119 * We use page writeout completions because we are interested in getting rid of 120 * dirty pages. Having them written out is the primary goal. 121 * 122 * We introduce a concept of time, a period over which we measure these events, 123 * because demand can/will vary over time. The length of this period itself is 124 * measured in page writeback completions. 125 * 126 */ 127static struct prop_descriptor vm_completions; 128static struct prop_descriptor vm_dirties; 129 130/* 131 * couple the period to the dirty_ratio: 132 * 133 * period/2 ~ roundup_pow_of_two(dirty limit) 134 */ 135static int calc_period_shift(void) 136{ 137 unsigned long dirty_total; 138 139 if (vm_dirty_bytes) 140 dirty_total = vm_dirty_bytes / PAGE_SIZE; 141 else 142 dirty_total = (vm_dirty_ratio * determine_dirtyable_memory()) / 143 100; 144 return 2 + ilog2(dirty_total - 1); 145} 146 147/* 148 * update the period when the dirty threshold changes. 149 */ 150static void update_completion_period(void) 151{ 152 int shift = calc_period_shift(); 153 prop_change_shift(&vm_completions, shift); 154 prop_change_shift(&vm_dirties, shift); 155} 156 157int dirty_background_ratio_handler(struct ctl_table *table, int write, 158 struct file *filp, void __user *buffer, size_t *lenp, 159 loff_t *ppos) 160{ 161 int ret; 162 163 ret = proc_dointvec_minmax(table, write, filp, buffer, lenp, ppos); 164 if (ret == 0 && write) 165 dirty_background_bytes = 0; 166 return ret; 167} 168 169int dirty_background_bytes_handler(struct ctl_table *table, int write, 170 struct file *filp, void __user *buffer, size_t *lenp, 171 loff_t *ppos) 172{ 173 int ret; 174 175 ret = proc_doulongvec_minmax(table, write, filp, buffer, lenp, ppos); 176 if (ret == 0 && write) 177 dirty_background_ratio = 0; 178 return ret; 179} 180 181int dirty_ratio_handler(struct ctl_table *table, int write, 182 struct file *filp, void __user *buffer, size_t *lenp, 183 loff_t *ppos) 184{ 185 int old_ratio = vm_dirty_ratio; 186 int ret; 187 188 ret = proc_dointvec_minmax(table, write, filp, buffer, lenp, ppos); 189 if (ret == 0 && write && vm_dirty_ratio != old_ratio) { 190 update_completion_period(); 191 vm_dirty_bytes = 0; 192 } 193 return ret; 194} 195 196 197int dirty_bytes_handler(struct ctl_table *table, int write, 198 struct file *filp, void __user *buffer, size_t *lenp, 199 loff_t *ppos) 200{ 201 unsigned long old_bytes = vm_dirty_bytes; 202 int ret; 203 204 ret = proc_doulongvec_minmax(table, write, filp, buffer, lenp, ppos); 205 if (ret == 0 && write && vm_dirty_bytes != old_bytes) { 206 update_completion_period(); 207 vm_dirty_ratio = 0; 208 } 209 return ret; 210} 211 212/* 213 * Increment the BDI's writeout completion count and the global writeout 214 * completion count. Called from test_clear_page_writeback(). 215 */ 216static inline void __bdi_writeout_inc(struct backing_dev_info *bdi) 217{ 218 __prop_inc_percpu_max(&vm_completions, &bdi->completions, 219 bdi->max_prop_frac); 220} 221 222void bdi_writeout_inc(struct backing_dev_info *bdi) 223{ 224 unsigned long flags; 225 226 local_irq_save(flags); 227 __bdi_writeout_inc(bdi); 228 local_irq_restore(flags); 229} 230EXPORT_SYMBOL_GPL(bdi_writeout_inc); 231 232void task_dirty_inc(struct task_struct *tsk) 233{ 234 prop_inc_single(&vm_dirties, &tsk->dirties); 235} 236 237/* 238 * Obtain an accurate fraction of the BDI's portion. 239 */ 240static void bdi_writeout_fraction(struct backing_dev_info *bdi, 241 long *numerator, long *denominator) 242{ 243 if (bdi_cap_writeback_dirty(bdi)) { 244 prop_fraction_percpu(&vm_completions, &bdi->completions, 245 numerator, denominator); 246 } else { 247 *numerator = 0; 248 *denominator = 1; 249 } 250} 251 252/* 253 * Clip the earned share of dirty pages to that which is actually available. 254 * This avoids exceeding the total dirty_limit when the floating averages 255 * fluctuate too quickly. 256 */ 257static void clip_bdi_dirty_limit(struct backing_dev_info *bdi, 258 unsigned long dirty, unsigned long *pbdi_dirty) 259{ 260 unsigned long avail_dirty; 261 262 avail_dirty = global_page_state(NR_FILE_DIRTY) + 263 global_page_state(NR_WRITEBACK) + 264 global_page_state(NR_UNSTABLE_NFS) + 265 global_page_state(NR_WRITEBACK_TEMP); 266 267 if (avail_dirty < dirty) 268 avail_dirty = dirty - avail_dirty; 269 else 270 avail_dirty = 0; 271 272 avail_dirty += bdi_stat(bdi, BDI_RECLAIMABLE) + 273 bdi_stat(bdi, BDI_WRITEBACK); 274 275 *pbdi_dirty = min(*pbdi_dirty, avail_dirty); 276} 277 278static inline void task_dirties_fraction(struct task_struct *tsk, 279 long *numerator, long *denominator) 280{ 281 prop_fraction_single(&vm_dirties, &tsk->dirties, 282 numerator, denominator); 283} 284 285/* 286 * scale the dirty limit 287 * 288 * task specific dirty limit: 289 * 290 * dirty -= (dirty/8) * p_{t} 291 */ 292static void task_dirty_limit(struct task_struct *tsk, unsigned long *pdirty) 293{ 294 long numerator, denominator; 295 unsigned long dirty = *pdirty; 296 u64 inv = dirty >> 3; 297 298 task_dirties_fraction(tsk, &numerator, &denominator); 299 inv *= numerator; 300 do_div(inv, denominator); 301 302 dirty -= inv; 303 if (dirty < *pdirty/2) 304 dirty = *pdirty/2; 305 306 *pdirty = dirty; 307} 308 309/* 310 * 311 */ 312static unsigned int bdi_min_ratio; 313 314int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio) 315{ 316 int ret = 0; 317 318 spin_lock(&bdi_lock); 319 if (min_ratio > bdi->max_ratio) { 320 ret = -EINVAL; 321 } else { 322 min_ratio -= bdi->min_ratio; 323 if (bdi_min_ratio + min_ratio < 100) { 324 bdi_min_ratio += min_ratio; 325 bdi->min_ratio += min_ratio; 326 } else { 327 ret = -EINVAL; 328 } 329 } 330 spin_unlock(&bdi_lock); 331 332 return ret; 333} 334 335int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned max_ratio) 336{ 337 int ret = 0; 338 339 if (max_ratio > 100) 340 return -EINVAL; 341 342 spin_lock(&bdi_lock); 343 if (bdi->min_ratio > max_ratio) { 344 ret = -EINVAL; 345 } else { 346 bdi->max_ratio = max_ratio; 347 bdi->max_prop_frac = (PROP_FRAC_BASE * max_ratio) / 100; 348 } 349 spin_unlock(&bdi_lock); 350 351 return ret; 352} 353EXPORT_SYMBOL(bdi_set_max_ratio); 354 355/* 356 * Work out the current dirty-memory clamping and background writeout 357 * thresholds. 358 * 359 * The main aim here is to lower them aggressively if there is a lot of mapped 360 * memory around. To avoid stressing page reclaim with lots of unreclaimable 361 * pages. It is better to clamp down on writers than to start swapping, and 362 * performing lots of scanning. 363 * 364 * We only allow 1/2 of the currently-unmapped memory to be dirtied. 365 * 366 * We don't permit the clamping level to fall below 5% - that is getting rather 367 * excessive. 368 * 369 * We make sure that the background writeout level is below the adjusted 370 * clamping level. 371 */ 372 373static unsigned long highmem_dirtyable_memory(unsigned long total) 374{ 375#ifdef CONFIG_HIGHMEM 376 int node; 377 unsigned long x = 0; 378 379 for_each_node_state(node, N_HIGH_MEMORY) { 380 struct zone *z = 381 &NODE_DATA(node)->node_zones[ZONE_HIGHMEM]; 382 383 x += zone_page_state(z, NR_FREE_PAGES) + zone_lru_pages(z); 384 } 385 /* 386 * Make sure that the number of highmem pages is never larger 387 * than the number of the total dirtyable memory. This can only 388 * occur in very strange VM situations but we want to make sure 389 * that this does not occur. 390 */ 391 return min(x, total); 392#else 393 return 0; 394#endif 395} 396 397/** 398 * determine_dirtyable_memory - amount of memory that may be used 399 * 400 * Returns the numebr of pages that can currently be freed and used 401 * by the kernel for direct mappings. 402 */ 403unsigned long determine_dirtyable_memory(void) 404{ 405 unsigned long x; 406 407 x = global_page_state(NR_FREE_PAGES) + global_lru_pages(); 408 409 if (!vm_highmem_is_dirtyable) 410 x -= highmem_dirtyable_memory(x); 411 412 return x + 1; /* Ensure that we never return 0 */ 413} 414 415void 416get_dirty_limits(unsigned long *pbackground, unsigned long *pdirty, 417 unsigned long *pbdi_dirty, struct backing_dev_info *bdi) 418{ 419 unsigned long background; 420 unsigned long dirty; 421 unsigned long available_memory = determine_dirtyable_memory(); 422 struct task_struct *tsk; 423 424 if (vm_dirty_bytes) 425 dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE); 426 else { 427 int dirty_ratio; 428 429 dirty_ratio = vm_dirty_ratio; 430 if (dirty_ratio < 5) 431 dirty_ratio = 5; 432 dirty = (dirty_ratio * available_memory) / 100; 433 } 434 435 if (dirty_background_bytes) 436 background = DIV_ROUND_UP(dirty_background_bytes, PAGE_SIZE); 437 else 438 background = (dirty_background_ratio * available_memory) / 100; 439 440 if (background >= dirty) 441 background = dirty / 2; 442 tsk = current; 443 if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk)) { 444 background += background / 4; 445 dirty += dirty / 4; 446 } 447 *pbackground = background; 448 *pdirty = dirty; 449 450 if (bdi) { 451 u64 bdi_dirty; 452 long numerator, denominator; 453 454 /* 455 * Calculate this BDI's share of the dirty ratio. 456 */ 457 bdi_writeout_fraction(bdi, &numerator, &denominator); 458 459 bdi_dirty = (dirty * (100 - bdi_min_ratio)) / 100; 460 bdi_dirty *= numerator; 461 do_div(bdi_dirty, denominator); 462 bdi_dirty += (dirty * bdi->min_ratio) / 100; 463 if (bdi_dirty > (dirty * bdi->max_ratio) / 100) 464 bdi_dirty = dirty * bdi->max_ratio / 100; 465 466 *pbdi_dirty = bdi_dirty; 467 clip_bdi_dirty_limit(bdi, dirty, pbdi_dirty); 468 task_dirty_limit(current, pbdi_dirty); 469 } 470} 471 472/* 473 * balance_dirty_pages() must be called by processes which are generating dirty 474 * data. It looks at the number of dirty pages in the machine and will force 475 * the caller to perform writeback if the system is over `vm_dirty_ratio'. 476 * If we're over `background_thresh' then pdflush is woken to perform some 477 * writeout. 478 */ 479static void balance_dirty_pages(struct address_space *mapping) 480{ 481 long nr_reclaimable, bdi_nr_reclaimable; 482 long nr_writeback, bdi_nr_writeback; 483 unsigned long background_thresh; 484 unsigned long dirty_thresh; 485 unsigned long bdi_thresh; 486 unsigned long pages_written = 0; 487 unsigned long write_chunk = sync_writeback_pages(); 488 489 struct backing_dev_info *bdi = mapping->backing_dev_info; 490 491 for (;;) { 492 struct writeback_control wbc = { 493 .bdi = bdi, 494 .sync_mode = WB_SYNC_NONE, 495 .older_than_this = NULL, 496 .nr_to_write = write_chunk, 497 .range_cyclic = 1, 498 }; 499 500 get_dirty_limits(&background_thresh, &dirty_thresh, 501 &bdi_thresh, bdi); 502 503 nr_reclaimable = global_page_state(NR_FILE_DIRTY) + 504 global_page_state(NR_UNSTABLE_NFS); 505 nr_writeback = global_page_state(NR_WRITEBACK); 506 507 bdi_nr_reclaimable = bdi_stat(bdi, BDI_RECLAIMABLE); 508 bdi_nr_writeback = bdi_stat(bdi, BDI_WRITEBACK); 509 510 if (bdi_nr_reclaimable + bdi_nr_writeback <= bdi_thresh) 511 break; 512 513 /* 514 * Throttle it only when the background writeback cannot 515 * catch-up. This avoids (excessively) small writeouts 516 * when the bdi limits are ramping up. 517 */ 518 if (nr_reclaimable + nr_writeback < 519 (background_thresh + dirty_thresh) / 2) 520 break; 521 522 if (!bdi->dirty_exceeded) 523 bdi->dirty_exceeded = 1; 524 525 /* Note: nr_reclaimable denotes nr_dirty + nr_unstable. 526 * Unstable writes are a feature of certain networked 527 * filesystems (i.e. NFS) in which data may have been 528 * written to the server's write cache, but has not yet 529 * been flushed to permanent storage. 530 * Only move pages to writeback if this bdi is over its 531 * threshold otherwise wait until the disk writes catch 532 * up. 533 */ 534 if (bdi_nr_reclaimable > bdi_thresh) { 535 writeback_inodes_wbc(&wbc); 536 pages_written += write_chunk - wbc.nr_to_write; 537 get_dirty_limits(&background_thresh, &dirty_thresh, 538 &bdi_thresh, bdi); 539 } 540 541 /* 542 * In order to avoid the stacked BDI deadlock we need 543 * to ensure we accurately count the 'dirty' pages when 544 * the threshold is low. 545 * 546 * Otherwise it would be possible to get thresh+n pages 547 * reported dirty, even though there are thresh-m pages 548 * actually dirty; with m+n sitting in the percpu 549 * deltas. 550 */ 551 if (bdi_thresh < 2*bdi_stat_error(bdi)) { 552 bdi_nr_reclaimable = bdi_stat_sum(bdi, BDI_RECLAIMABLE); 553 bdi_nr_writeback = bdi_stat_sum(bdi, BDI_WRITEBACK); 554 } else if (bdi_nr_reclaimable) { 555 bdi_nr_reclaimable = bdi_stat(bdi, BDI_RECLAIMABLE); 556 bdi_nr_writeback = bdi_stat(bdi, BDI_WRITEBACK); 557 } 558 559 if (bdi_nr_reclaimable + bdi_nr_writeback <= bdi_thresh) 560 break; 561 if (pages_written >= write_chunk) 562 break; /* We've done our duty */ 563 564 schedule_timeout(1); 565 } 566 567 if (bdi_nr_reclaimable + bdi_nr_writeback < bdi_thresh && 568 bdi->dirty_exceeded) 569 bdi->dirty_exceeded = 0; 570 571 if (writeback_in_progress(bdi)) 572 return; /* pdflush is already working this queue */ 573 574 /* 575 * In laptop mode, we wait until hitting the higher threshold before 576 * starting background writeout, and then write out all the way down 577 * to the lower threshold. So slow writers cause minimal disk activity. 578 * 579 * In normal mode, we start background writeout at the lower 580 * background_thresh, to keep the amount of dirty memory low. 581 */ 582 if ((laptop_mode && pages_written) || 583 (!laptop_mode && ((nr_writeback = global_page_state(NR_FILE_DIRTY) 584 + global_page_state(NR_UNSTABLE_NFS)) 585 > background_thresh))) { 586 struct writeback_control wbc = { 587 .bdi = bdi, 588 .sync_mode = WB_SYNC_NONE, 589 .nr_to_write = nr_writeback, 590 }; 591 592 593 bdi_start_writeback(&wbc); 594 } 595} 596 597void set_page_dirty_balance(struct page *page, int page_mkwrite) 598{ 599 if (set_page_dirty(page) || page_mkwrite) { 600 struct address_space *mapping = page_mapping(page); 601 602 if (mapping) 603 balance_dirty_pages_ratelimited(mapping); 604 } 605} 606 607static DEFINE_PER_CPU(unsigned long, bdp_ratelimits) = 0; 608 609/** 610 * balance_dirty_pages_ratelimited_nr - balance dirty memory state 611 * @mapping: address_space which was dirtied 612 * @nr_pages_dirtied: number of pages which the caller has just dirtied 613 * 614 * Processes which are dirtying memory should call in here once for each page 615 * which was newly dirtied. The function will periodically check the system's 616 * dirty state and will initiate writeback if needed. 617 * 618 * On really big machines, get_writeback_state is expensive, so try to avoid 619 * calling it too often (ratelimiting). But once we're over the dirty memory 620 * limit we decrease the ratelimiting by a lot, to prevent individual processes 621 * from overshooting the limit by (ratelimit_pages) each. 622 */ 623void balance_dirty_pages_ratelimited_nr(struct address_space *mapping, 624 unsigned long nr_pages_dirtied) 625{ 626 unsigned long ratelimit; 627 unsigned long *p; 628 629 ratelimit = ratelimit_pages; 630 if (mapping->backing_dev_info->dirty_exceeded) 631 ratelimit = 8; 632 633 /* 634 * Check the rate limiting. Also, we do not want to throttle real-time 635 * tasks in balance_dirty_pages(). Period. 636 */ 637 preempt_disable(); 638 p = &__get_cpu_var(bdp_ratelimits); 639 *p += nr_pages_dirtied; 640 if (unlikely(*p >= ratelimit)) { 641 *p = 0; 642 preempt_enable(); 643 balance_dirty_pages(mapping); 644 return; 645 } 646 preempt_enable(); 647} 648EXPORT_SYMBOL(balance_dirty_pages_ratelimited_nr); 649 650void throttle_vm_writeout(gfp_t gfp_mask) 651{ 652 unsigned long background_thresh; 653 unsigned long dirty_thresh; 654 655 for ( ; ; ) { 656 get_dirty_limits(&background_thresh, &dirty_thresh, NULL, NULL); 657 658 /* 659 * Boost the allowable dirty threshold a bit for page 660 * allocators so they don't get DoS'ed by heavy writers 661 */ 662 dirty_thresh += dirty_thresh / 10; /* wheeee... */ 663 664 if (global_page_state(NR_UNSTABLE_NFS) + 665 global_page_state(NR_WRITEBACK) <= dirty_thresh) 666 break; 667 congestion_wait(BLK_RW_ASYNC, HZ/10); 668 669 /* 670 * The caller might hold locks which can prevent IO completion 671 * or progress in the filesystem. So we cannot just sit here 672 * waiting for IO to complete. 673 */ 674 if ((gfp_mask & (__GFP_FS|__GFP_IO)) != (__GFP_FS|__GFP_IO)) 675 break; 676 } 677} 678 679static void laptop_timer_fn(unsigned long unused); 680 681static DEFINE_TIMER(laptop_mode_wb_timer, laptop_timer_fn, 0, 0); 682 683/* 684 * sysctl handler for /proc/sys/vm/dirty_writeback_centisecs 685 */ 686int dirty_writeback_centisecs_handler(ctl_table *table, int write, 687 struct file *file, void __user *buffer, size_t *length, loff_t *ppos) 688{ 689 proc_dointvec(table, write, file, buffer, length, ppos); 690 return 0; 691} 692 693static void do_laptop_sync(struct work_struct *work) 694{ 695 wakeup_flusher_threads(0); 696 kfree(work); 697} 698 699static void laptop_timer_fn(unsigned long unused) 700{ 701 struct work_struct *work; 702 703 work = kmalloc(sizeof(*work), GFP_ATOMIC); 704 if (work) { 705 INIT_WORK(work, do_laptop_sync); 706 schedule_work(work); 707 } 708} 709 710/* 711 * We've spun up the disk and we're in laptop mode: schedule writeback 712 * of all dirty data a few seconds from now. If the flush is already scheduled 713 * then push it back - the user is still using the disk. 714 */ 715void laptop_io_completion(void) 716{ 717 mod_timer(&laptop_mode_wb_timer, jiffies + laptop_mode); 718} 719 720/* 721 * We're in laptop mode and we've just synced. The sync's writes will have 722 * caused another writeback to be scheduled by laptop_io_completion. 723 * Nothing needs to be written back anymore, so we unschedule the writeback. 724 */ 725void laptop_sync_completion(void) 726{ 727 del_timer(&laptop_mode_wb_timer); 728} 729 730/* 731 * If ratelimit_pages is too high then we can get into dirty-data overload 732 * if a large number of processes all perform writes at the same time. 733 * If it is too low then SMP machines will call the (expensive) 734 * get_writeback_state too often. 735 * 736 * Here we set ratelimit_pages to a level which ensures that when all CPUs are 737 * dirtying in parallel, we cannot go more than 3% (1/32) over the dirty memory 738 * thresholds before writeback cuts in. 739 * 740 * But the limit should not be set too high. Because it also controls the 741 * amount of memory which the balance_dirty_pages() caller has to write back. 742 * If this is too large then the caller will block on the IO queue all the 743 * time. So limit it to four megabytes - the balance_dirty_pages() caller 744 * will write six megabyte chunks, max. 745 */ 746 747void writeback_set_ratelimit(void) 748{ 749 ratelimit_pages = vm_total_pages / (num_online_cpus() * 32); 750 if (ratelimit_pages < 16) 751 ratelimit_pages = 16; 752 if (ratelimit_pages * PAGE_CACHE_SIZE > 4096 * 1024) 753 ratelimit_pages = (4096 * 1024) / PAGE_CACHE_SIZE; 754} 755 756static int __cpuinit 757ratelimit_handler(struct notifier_block *self, unsigned long u, void *v) 758{ 759 writeback_set_ratelimit(); 760 return NOTIFY_DONE; 761} 762 763static struct notifier_block __cpuinitdata ratelimit_nb = { 764 .notifier_call = ratelimit_handler, 765 .next = NULL, 766}; 767 768/* 769 * Called early on to tune the page writeback dirty limits. 770 * 771 * We used to scale dirty pages according to how total memory 772 * related to pages that could be allocated for buffers (by 773 * comparing nr_free_buffer_pages() to vm_total_pages. 774 * 775 * However, that was when we used "dirty_ratio" to scale with 776 * all memory, and we don't do that any more. "dirty_ratio" 777 * is now applied to total non-HIGHPAGE memory (by subtracting 778 * totalhigh_pages from vm_total_pages), and as such we can't 779 * get into the old insane situation any more where we had 780 * large amounts of dirty pages compared to a small amount of 781 * non-HIGHMEM memory. 782 * 783 * But we might still want to scale the dirty_ratio by how 784 * much memory the box has.. 785 */ 786void __init page_writeback_init(void) 787{ 788 int shift; 789 790 writeback_set_ratelimit(); 791 register_cpu_notifier(&ratelimit_nb); 792 793 shift = calc_period_shift(); 794 prop_descriptor_init(&vm_completions, shift); 795 prop_descriptor_init(&vm_dirties, shift); 796} 797 798/** 799 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them. 800 * @mapping: address space structure to write 801 * @wbc: subtract the number of written pages from *@wbc->nr_to_write 802 * @writepage: function called for each page 803 * @data: data passed to writepage function 804 * 805 * If a page is already under I/O, write_cache_pages() skips it, even 806 * if it's dirty. This is desirable behaviour for memory-cleaning writeback, 807 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync() 808 * and msync() need to guarantee that all the data which was dirty at the time 809 * the call was made get new I/O started against them. If wbc->sync_mode is 810 * WB_SYNC_ALL then we were called for data integrity and we must wait for 811 * existing IO to complete. 812 */ 813int write_cache_pages(struct address_space *mapping, 814 struct writeback_control *wbc, writepage_t writepage, 815 void *data) 816{ 817 struct backing_dev_info *bdi = mapping->backing_dev_info; 818 int ret = 0; 819 int done = 0; 820 struct pagevec pvec; 821 int nr_pages; 822 pgoff_t uninitialized_var(writeback_index); 823 pgoff_t index; 824 pgoff_t end; /* Inclusive */ 825 pgoff_t done_index; 826 int cycled; 827 int range_whole = 0; 828 long nr_to_write = wbc->nr_to_write; 829 830 if (wbc->nonblocking && bdi_write_congested(bdi)) { 831 wbc->encountered_congestion = 1; 832 return 0; 833 } 834 835 pagevec_init(&pvec, 0); 836 if (wbc->range_cyclic) { 837 writeback_index = mapping->writeback_index; /* prev offset */ 838 index = writeback_index; 839 if (index == 0) 840 cycled = 1; 841 else 842 cycled = 0; 843 end = -1; 844 } else { 845 index = wbc->range_start >> PAGE_CACHE_SHIFT; 846 end = wbc->range_end >> PAGE_CACHE_SHIFT; 847 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 848 range_whole = 1; 849 cycled = 1; /* ignore range_cyclic tests */ 850 } 851retry: 852 done_index = index; 853 while (!done && (index <= end)) { 854 int i; 855 856 nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, 857 PAGECACHE_TAG_DIRTY, 858 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1); 859 if (nr_pages == 0) 860 break; 861 862 for (i = 0; i < nr_pages; i++) { 863 struct page *page = pvec.pages[i]; 864 865 /* 866 * At this point, the page may be truncated or 867 * invalidated (changing page->mapping to NULL), or 868 * even swizzled back from swapper_space to tmpfs file 869 * mapping. However, page->index will not change 870 * because we have a reference on the page. 871 */ 872 if (page->index > end) { 873 /* 874 * can't be range_cyclic (1st pass) because 875 * end == -1 in that case. 876 */ 877 done = 1; 878 break; 879 } 880 881 done_index = page->index + 1; 882 883 lock_page(page); 884 885 /* 886 * Page truncated or invalidated. We can freely skip it 887 * then, even for data integrity operations: the page 888 * has disappeared concurrently, so there could be no 889 * real expectation of this data interity operation 890 * even if there is now a new, dirty page at the same 891 * pagecache address. 892 */ 893 if (unlikely(page->mapping != mapping)) { 894continue_unlock: 895 unlock_page(page); 896 continue; 897 } 898 899 if (!PageDirty(page)) { 900 /* someone wrote it for us */ 901 goto continue_unlock; 902 } 903 904 if (PageWriteback(page)) { 905 if (wbc->sync_mode != WB_SYNC_NONE) 906 wait_on_page_writeback(page); 907 else 908 goto continue_unlock; 909 } 910 911 BUG_ON(PageWriteback(page)); 912 if (!clear_page_dirty_for_io(page)) 913 goto continue_unlock; 914 915 ret = (*writepage)(page, wbc, data); 916 if (unlikely(ret)) { 917 if (ret == AOP_WRITEPAGE_ACTIVATE) { 918 unlock_page(page); 919 ret = 0; 920 } else { 921 /* 922 * done_index is set past this page, 923 * so media errors will not choke 924 * background writeout for the entire 925 * file. This has consequences for 926 * range_cyclic semantics (ie. it may 927 * not be suitable for data integrity 928 * writeout). 929 */ 930 done = 1; 931 break; 932 } 933 } 934 935 if (nr_to_write > 0) { 936 nr_to_write--; 937 if (nr_to_write == 0 && 938 wbc->sync_mode == WB_SYNC_NONE) { 939 /* 940 * We stop writing back only if we are 941 * not doing integrity sync. In case of 942 * integrity sync we have to keep going 943 * because someone may be concurrently 944 * dirtying pages, and we might have 945 * synced a lot of newly appeared dirty 946 * pages, but have not synced all of the 947 * old dirty pages. 948 */ 949 done = 1; 950 break; 951 } 952 } 953 954 if (wbc->nonblocking && bdi_write_congested(bdi)) { 955 wbc->encountered_congestion = 1; 956 done = 1; 957 break; 958 } 959 } 960 pagevec_release(&pvec); 961 cond_resched(); 962 } 963 if (!cycled && !done) { 964 /* 965 * range_cyclic: 966 * We hit the last page and there is more work to be done: wrap 967 * back to the start of the file 968 */ 969 cycled = 1; 970 index = 0; 971 end = writeback_index - 1; 972 goto retry; 973 } 974 if (!wbc->no_nrwrite_index_update) { 975 if (wbc->range_cyclic || (range_whole && nr_to_write > 0)) 976 mapping->writeback_index = done_index; 977 wbc->nr_to_write = nr_to_write; 978 } 979 980 return ret; 981} 982EXPORT_SYMBOL(write_cache_pages); 983 984/* 985 * Function used by generic_writepages to call the real writepage 986 * function and set the mapping flags on error 987 */ 988static int __writepage(struct page *page, struct writeback_control *wbc, 989 void *data) 990{ 991 struct address_space *mapping = data; 992 int ret = mapping->a_ops->writepage(page, wbc); 993 mapping_set_error(mapping, ret); 994 return ret; 995} 996 997/** 998 * generic_writepages - walk the list of dirty pages of the given address space and writepage() all of them. 999 * @mapping: address space structure to write 1000 * @wbc: subtract the number of written pages from *@wbc->nr_to_write 1001 * 1002 * This is a library function, which implements the writepages() 1003 * address_space_operation. 1004 */ 1005int generic_writepages(struct address_space *mapping, 1006 struct writeback_control *wbc) 1007{ 1008 /* deal with chardevs and other special file */ 1009 if (!mapping->a_ops->writepage) 1010 return 0; 1011 1012 return write_cache_pages(mapping, wbc, __writepage, mapping); 1013} 1014 1015EXPORT_SYMBOL(generic_writepages); 1016 1017int do_writepages(struct address_space *mapping, struct writeback_control *wbc) 1018{ 1019 int ret; 1020 1021 if (wbc->nr_to_write <= 0) 1022 return 0; 1023 if (mapping->a_ops->writepages) 1024 ret = mapping->a_ops->writepages(mapping, wbc); 1025 else 1026 ret = generic_writepages(mapping, wbc); 1027 return ret; 1028} 1029 1030/** 1031 * write_one_page - write out a single page and optionally wait on I/O 1032 * @page: the page to write 1033 * @wait: if true, wait on writeout 1034 * 1035 * The page must be locked by the caller and will be unlocked upon return. 1036 * 1037 * write_one_page() returns a negative error code if I/O failed. 1038 */ 1039int write_one_page(struct page *page, int wait) 1040{ 1041 struct address_space *mapping = page->mapping; 1042 int ret = 0; 1043 struct writeback_control wbc = { 1044 .sync_mode = WB_SYNC_ALL, 1045 .nr_to_write = 1, 1046 }; 1047 1048 BUG_ON(!PageLocked(page)); 1049 1050 if (wait) 1051 wait_on_page_writeback(page); 1052 1053 if (clear_page_dirty_for_io(page)) { 1054 page_cache_get(page); 1055 ret = mapping->a_ops->writepage(page, &wbc); 1056 if (ret == 0 && wait) { 1057 wait_on_page_writeback(page); 1058 if (PageError(page)) 1059 ret = -EIO; 1060 } 1061 page_cache_release(page); 1062 } else { 1063 unlock_page(page); 1064 } 1065 return ret; 1066} 1067EXPORT_SYMBOL(write_one_page); 1068 1069/* 1070 * For address_spaces which do not use buffers nor write back. 1071 */ 1072int __set_page_dirty_no_writeback(struct page *page) 1073{ 1074 if (!PageDirty(page)) 1075 SetPageDirty(page); 1076 return 0; 1077} 1078 1079/* 1080 * Helper function for set_page_dirty family. 1081 * NOTE: This relies on being atomic wrt interrupts. 1082 */ 1083void account_page_dirtied(struct page *page, struct address_space *mapping) 1084{ 1085 if (mapping_cap_account_dirty(mapping)) { 1086 __inc_zone_page_state(page, NR_FILE_DIRTY); 1087 __inc_bdi_stat(mapping->backing_dev_info, BDI_RECLAIMABLE); 1088 task_dirty_inc(current); 1089 task_io_account_write(PAGE_CACHE_SIZE); 1090 } 1091} 1092 1093/* 1094 * For address_spaces which do not use buffers. Just tag the page as dirty in 1095 * its radix tree. 1096 * 1097 * This is also used when a single buffer is being dirtied: we want to set the 1098 * page dirty in that case, but not all the buffers. This is a "bottom-up" 1099 * dirtying, whereas __set_page_dirty_buffers() is a "top-down" dirtying. 1100 * 1101 * Most callers have locked the page, which pins the address_space in memory. 1102 * But zap_pte_range() does not lock the page, however in that case the 1103 * mapping is pinned by the vma's ->vm_file reference. 1104 * 1105 * We take care to handle the case where the page was truncated from the 1106 * mapping by re-checking page_mapping() inside tree_lock. 1107 */ 1108int __set_page_dirty_nobuffers(struct page *page) 1109{ 1110 if (!TestSetPageDirty(page)) { 1111 struct address_space *mapping = page_mapping(page); 1112 struct address_space *mapping2; 1113 1114 if (!mapping) 1115 return 1; 1116 1117 spin_lock_irq(&mapping->tree_lock); 1118 mapping2 = page_mapping(page); 1119 if (mapping2) { /* Race with truncate? */ 1120 BUG_ON(mapping2 != mapping); 1121 WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page)); 1122 account_page_dirtied(page, mapping); 1123 radix_tree_tag_set(&mapping->page_tree, 1124 page_index(page), PAGECACHE_TAG_DIRTY); 1125 } 1126 spin_unlock_irq(&mapping->tree_lock); 1127 if (mapping->host) { 1128 /* !PageAnon && !swapper_space */ 1129 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); 1130 } 1131 return 1; 1132 } 1133 return 0; 1134} 1135EXPORT_SYMBOL(__set_page_dirty_nobuffers); 1136 1137/* 1138 * When a writepage implementation decides that it doesn't want to write this 1139 * page for some reason, it should redirty the locked page via 1140 * redirty_page_for_writepage() and it should then unlock the page and return 0 1141 */ 1142int redirty_page_for_writepage(struct writeback_control *wbc, struct page *page) 1143{ 1144 wbc->pages_skipped++; 1145 return __set_page_dirty_nobuffers(page); 1146} 1147EXPORT_SYMBOL(redirty_page_for_writepage); 1148 1149/* 1150 * If the mapping doesn't provide a set_page_dirty a_op, then 1151 * just fall through and assume that it wants buffer_heads. 1152 */ 1153int set_page_dirty(struct page *page) 1154{ 1155 struct address_space *mapping = page_mapping(page); 1156 1157 if (likely(mapping)) { 1158 int (*spd)(struct page *) = mapping->a_ops->set_page_dirty; 1159#ifdef CONFIG_BLOCK 1160 if (!spd) 1161 spd = __set_page_dirty_buffers; 1162#endif 1163 return (*spd)(page); 1164 } 1165 if (!PageDirty(page)) { 1166 if (!TestSetPageDirty(page)) 1167 return 1; 1168 } 1169 return 0; 1170} 1171EXPORT_SYMBOL(set_page_dirty); 1172 1173/* 1174 * set_page_dirty() is racy if the caller has no reference against 1175 * page->mapping->host, and if the page is unlocked. This is because another 1176 * CPU could truncate the page off the mapping and then free the mapping. 1177 * 1178 * Usually, the page _is_ locked, or the caller is a user-space process which 1179 * holds a reference on the inode by having an open file. 1180 * 1181 * In other cases, the page should be locked before running set_page_dirty(). 1182 */ 1183int set_page_dirty_lock(struct page *page) 1184{ 1185 int ret; 1186 1187 lock_page_nosync(page); 1188 ret = set_page_dirty(page); 1189 unlock_page(page); 1190 return ret; 1191} 1192EXPORT_SYMBOL(set_page_dirty_lock); 1193 1194/* 1195 * Clear a page's dirty flag, while caring for dirty memory accounting. 1196 * Returns true if the page was previously dirty. 1197 * 1198 * This is for preparing to put the page under writeout. We leave the page 1199 * tagged as dirty in the radix tree so that a concurrent write-for-sync 1200 * can discover it via a PAGECACHE_TAG_DIRTY walk. The ->writepage 1201 * implementation will run either set_page_writeback() or set_page_dirty(), 1202 * at which stage we bring the page's dirty flag and radix-tree dirty tag 1203 * back into sync. 1204 * 1205 * This incoherency between the page's dirty flag and radix-tree tag is 1206 * unfortunate, but it only exists while the page is locked. 1207 */ 1208int clear_page_dirty_for_io(struct page *page) 1209{ 1210 struct address_space *mapping = page_mapping(page); 1211 1212 BUG_ON(!PageLocked(page)); 1213 1214 ClearPageReclaim(page); 1215 if (mapping && mapping_cap_account_dirty(mapping)) { 1216 /* 1217 * Yes, Virginia, this is indeed insane. 1218 * 1219 * We use this sequence to make sure that 1220 * (a) we account for dirty stats properly 1221 * (b) we tell the low-level filesystem to 1222 * mark the whole page dirty if it was 1223 * dirty in a pagetable. Only to then 1224 * (c) clean the page again and return 1 to 1225 * cause the writeback. 1226 * 1227 * This way we avoid all nasty races with the 1228 * dirty bit in multiple places and clearing 1229 * them concurrently from different threads. 1230 * 1231 * Note! Normally the "set_page_dirty(page)" 1232 * has no effect on the actual dirty bit - since 1233 * that will already usually be set. But we 1234 * need the side effects, and it can help us 1235 * avoid races. 1236 * 1237 * We basically use the page "master dirty bit" 1238 * as a serialization point for all the different 1239 * threads doing their things. 1240 */ 1241 if (page_mkclean(page)) 1242 set_page_dirty(page); 1243 /* 1244 * We carefully synchronise fault handlers against 1245 * installing a dirty pte and marking the page dirty 1246 * at this point. We do this by having them hold the 1247 * page lock at some point after installing their 1248 * pte, but before marking the page dirty. 1249 * Pages are always locked coming in here, so we get 1250 * the desired exclusion. See mm/memory.c:do_wp_page() 1251 * for more comments. 1252 */ 1253 if (TestClearPageDirty(page)) { 1254 dec_zone_page_state(page, NR_FILE_DIRTY); 1255 dec_bdi_stat(mapping->backing_dev_info, 1256 BDI_RECLAIMABLE); 1257 return 1; 1258 } 1259 return 0; 1260 } 1261 return TestClearPageDirty(page); 1262} 1263EXPORT_SYMBOL(clear_page_dirty_for_io); 1264 1265int test_clear_page_writeback(struct page *page) 1266{ 1267 struct address_space *mapping = page_mapping(page); 1268 int ret; 1269 1270 if (mapping) { 1271 struct backing_dev_info *bdi = mapping->backing_dev_info; 1272 unsigned long flags; 1273 1274 spin_lock_irqsave(&mapping->tree_lock, flags); 1275 ret = TestClearPageWriteback(page); 1276 if (ret) { 1277 radix_tree_tag_clear(&mapping->page_tree, 1278 page_index(page), 1279 PAGECACHE_TAG_WRITEBACK); 1280 if (bdi_cap_account_writeback(bdi)) { 1281 __dec_bdi_stat(bdi, BDI_WRITEBACK); 1282 __bdi_writeout_inc(bdi); 1283 } 1284 } 1285 spin_unlock_irqrestore(&mapping->tree_lock, flags); 1286 } else { 1287 ret = TestClearPageWriteback(page); 1288 } 1289 if (ret) 1290 dec_zone_page_state(page, NR_WRITEBACK); 1291 return ret; 1292} 1293 1294int test_set_page_writeback(struct page *page) 1295{ 1296 struct address_space *mapping = page_mapping(page); 1297 int ret; 1298 1299 if (mapping) { 1300 struct backing_dev_info *bdi = mapping->backing_dev_info; 1301 unsigned long flags; 1302 1303 spin_lock_irqsave(&mapping->tree_lock, flags); 1304 ret = TestSetPageWriteback(page); 1305 if (!ret) { 1306 radix_tree_tag_set(&mapping->page_tree, 1307 page_index(page), 1308 PAGECACHE_TAG_WRITEBACK); 1309 if (bdi_cap_account_writeback(bdi)) 1310 __inc_bdi_stat(bdi, BDI_WRITEBACK); 1311 } 1312 if (!PageDirty(page)) 1313 radix_tree_tag_clear(&mapping->page_tree, 1314 page_index(page), 1315 PAGECACHE_TAG_DIRTY); 1316 spin_unlock_irqrestore(&mapping->tree_lock, flags); 1317 } else { 1318 ret = TestSetPageWriteback(page); 1319 } 1320 if (!ret) 1321 inc_zone_page_state(page, NR_WRITEBACK); 1322 return ret; 1323 1324} 1325EXPORT_SYMBOL(test_set_page_writeback); 1326 1327/* 1328 * Return true if any of the pages in the mapping are marked with the 1329 * passed tag. 1330 */ 1331int mapping_tagged(struct address_space *mapping, int tag) 1332{ 1333 int ret; 1334 rcu_read_lock(); 1335 ret = radix_tree_tagged(&mapping->page_tree, tag); 1336 rcu_read_unlock(); 1337 return ret; 1338} 1339EXPORT_SYMBOL(mapping_tagged); 1340