page-writeback.c revision 07db59bd6b0f279c31044cba6787344f63be87ea
1/* 2 * mm/page-writeback.c 3 * 4 * Copyright (C) 2002, Linus Torvalds. 5 * 6 * Contains functions related to writing back dirty pages at the 7 * address_space level. 8 * 9 * 10Apr2002 akpm@zip.com.au 10 * Initial version 11 */ 12 13#include <linux/kernel.h> 14#include <linux/module.h> 15#include <linux/spinlock.h> 16#include <linux/fs.h> 17#include <linux/mm.h> 18#include <linux/swap.h> 19#include <linux/slab.h> 20#include <linux/pagemap.h> 21#include <linux/writeback.h> 22#include <linux/init.h> 23#include <linux/backing-dev.h> 24#include <linux/task_io_accounting_ops.h> 25#include <linux/blkdev.h> 26#include <linux/mpage.h> 27#include <linux/rmap.h> 28#include <linux/percpu.h> 29#include <linux/notifier.h> 30#include <linux/smp.h> 31#include <linux/sysctl.h> 32#include <linux/cpu.h> 33#include <linux/syscalls.h> 34#include <linux/buffer_head.h> 35#include <linux/pagevec.h> 36 37/* 38 * The maximum number of pages to writeout in a single bdflush/kupdate 39 * operation. We do this so we don't hold I_LOCK against an inode for 40 * enormous amounts of time, which would block a userspace task which has 41 * been forced to throttle against that inode. Also, the code reevaluates 42 * the dirty each time it has written this many pages. 43 */ 44#define MAX_WRITEBACK_PAGES 1024 45 46/* 47 * After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited 48 * will look to see if it needs to force writeback or throttling. 49 */ 50static long ratelimit_pages = 32; 51 52static int dirty_exceeded __cacheline_aligned_in_smp; /* Dirty mem may be over limit */ 53 54/* 55 * When balance_dirty_pages decides that the caller needs to perform some 56 * non-background writeback, this is how many pages it will attempt to write. 57 * It should be somewhat larger than RATELIMIT_PAGES to ensure that reasonably 58 * large amounts of I/O are submitted. 59 */ 60static inline long sync_writeback_pages(void) 61{ 62 return ratelimit_pages + ratelimit_pages / 2; 63} 64 65/* The following parameters are exported via /proc/sys/vm */ 66 67/* 68 * Start background writeback (via pdflush) at this percentage 69 */ 70int dirty_background_ratio = 5; 71 72/* 73 * The generator of dirty data starts writeback at this percentage 74 */ 75int vm_dirty_ratio = 10; 76 77/* 78 * The interval between `kupdate'-style writebacks, in jiffies 79 */ 80int dirty_writeback_interval = 5 * HZ; 81 82/* 83 * The longest number of jiffies for which data is allowed to remain dirty 84 */ 85int dirty_expire_interval = 30 * HZ; 86 87/* 88 * Flag that makes the machine dump writes/reads and block dirtyings. 89 */ 90int block_dump; 91 92/* 93 * Flag that puts the machine in "laptop mode". Doubles as a timeout in jiffies: 94 * a full sync is triggered after this time elapses without any disk activity. 95 */ 96int laptop_mode; 97 98EXPORT_SYMBOL(laptop_mode); 99 100/* End of sysctl-exported parameters */ 101 102 103static void background_writeout(unsigned long _min_pages); 104 105/* 106 * Work out the current dirty-memory clamping and background writeout 107 * thresholds. 108 * 109 * The main aim here is to lower them aggressively if there is a lot of mapped 110 * memory around. To avoid stressing page reclaim with lots of unreclaimable 111 * pages. It is better to clamp down on writers than to start swapping, and 112 * performing lots of scanning. 113 * 114 * We only allow 1/2 of the currently-unmapped memory to be dirtied. 115 * 116 * We don't permit the clamping level to fall below 5% - that is getting rather 117 * excessive. 118 * 119 * We make sure that the background writeout level is below the adjusted 120 * clamping level. 121 */ 122static void 123get_dirty_limits(long *pbackground, long *pdirty, 124 struct address_space *mapping) 125{ 126 int background_ratio; /* Percentages */ 127 int dirty_ratio; 128 int unmapped_ratio; 129 long background; 130 long dirty; 131 unsigned long available_memory = vm_total_pages; 132 struct task_struct *tsk; 133 134#ifdef CONFIG_HIGHMEM 135 /* 136 * We always exclude high memory from our count. 137 */ 138 available_memory -= totalhigh_pages; 139#endif 140 141 142 unmapped_ratio = 100 - ((global_page_state(NR_FILE_MAPPED) + 143 global_page_state(NR_ANON_PAGES)) * 100) / 144 vm_total_pages; 145 146 dirty_ratio = vm_dirty_ratio; 147 if (dirty_ratio > unmapped_ratio / 2) 148 dirty_ratio = unmapped_ratio / 2; 149 150 if (dirty_ratio < 5) 151 dirty_ratio = 5; 152 153 background_ratio = dirty_background_ratio; 154 if (background_ratio >= dirty_ratio) 155 background_ratio = dirty_ratio / 2; 156 157 background = (background_ratio * available_memory) / 100; 158 dirty = (dirty_ratio * available_memory) / 100; 159 tsk = current; 160 if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk)) { 161 background += background / 4; 162 dirty += dirty / 4; 163 } 164 *pbackground = background; 165 *pdirty = dirty; 166} 167 168/* 169 * balance_dirty_pages() must be called by processes which are generating dirty 170 * data. It looks at the number of dirty pages in the machine and will force 171 * the caller to perform writeback if the system is over `vm_dirty_ratio'. 172 * If we're over `background_thresh' then pdflush is woken to perform some 173 * writeout. 174 */ 175static void balance_dirty_pages(struct address_space *mapping) 176{ 177 long nr_reclaimable; 178 long background_thresh; 179 long dirty_thresh; 180 unsigned long pages_written = 0; 181 unsigned long write_chunk = sync_writeback_pages(); 182 183 struct backing_dev_info *bdi = mapping->backing_dev_info; 184 185 for (;;) { 186 struct writeback_control wbc = { 187 .bdi = bdi, 188 .sync_mode = WB_SYNC_NONE, 189 .older_than_this = NULL, 190 .nr_to_write = write_chunk, 191 .range_cyclic = 1, 192 }; 193 194 get_dirty_limits(&background_thresh, &dirty_thresh, mapping); 195 nr_reclaimable = global_page_state(NR_FILE_DIRTY) + 196 global_page_state(NR_UNSTABLE_NFS); 197 if (nr_reclaimable + global_page_state(NR_WRITEBACK) <= 198 dirty_thresh) 199 break; 200 201 if (!dirty_exceeded) 202 dirty_exceeded = 1; 203 204 /* Note: nr_reclaimable denotes nr_dirty + nr_unstable. 205 * Unstable writes are a feature of certain networked 206 * filesystems (i.e. NFS) in which data may have been 207 * written to the server's write cache, but has not yet 208 * been flushed to permanent storage. 209 */ 210 if (nr_reclaimable) { 211 writeback_inodes(&wbc); 212 get_dirty_limits(&background_thresh, 213 &dirty_thresh, mapping); 214 nr_reclaimable = global_page_state(NR_FILE_DIRTY) + 215 global_page_state(NR_UNSTABLE_NFS); 216 if (nr_reclaimable + 217 global_page_state(NR_WRITEBACK) 218 <= dirty_thresh) 219 break; 220 pages_written += write_chunk - wbc.nr_to_write; 221 if (pages_written >= write_chunk) 222 break; /* We've done our duty */ 223 } 224 congestion_wait(WRITE, HZ/10); 225 } 226 227 if (nr_reclaimable + global_page_state(NR_WRITEBACK) 228 <= dirty_thresh && dirty_exceeded) 229 dirty_exceeded = 0; 230 231 if (writeback_in_progress(bdi)) 232 return; /* pdflush is already working this queue */ 233 234 /* 235 * In laptop mode, we wait until hitting the higher threshold before 236 * starting background writeout, and then write out all the way down 237 * to the lower threshold. So slow writers cause minimal disk activity. 238 * 239 * In normal mode, we start background writeout at the lower 240 * background_thresh, to keep the amount of dirty memory low. 241 */ 242 if ((laptop_mode && pages_written) || 243 (!laptop_mode && (nr_reclaimable > background_thresh))) 244 pdflush_operation(background_writeout, 0); 245} 246 247void set_page_dirty_balance(struct page *page) 248{ 249 if (set_page_dirty(page)) { 250 struct address_space *mapping = page_mapping(page); 251 252 if (mapping) 253 balance_dirty_pages_ratelimited(mapping); 254 } 255} 256 257/** 258 * balance_dirty_pages_ratelimited_nr - balance dirty memory state 259 * @mapping: address_space which was dirtied 260 * @nr_pages_dirtied: number of pages which the caller has just dirtied 261 * 262 * Processes which are dirtying memory should call in here once for each page 263 * which was newly dirtied. The function will periodically check the system's 264 * dirty state and will initiate writeback if needed. 265 * 266 * On really big machines, get_writeback_state is expensive, so try to avoid 267 * calling it too often (ratelimiting). But once we're over the dirty memory 268 * limit we decrease the ratelimiting by a lot, to prevent individual processes 269 * from overshooting the limit by (ratelimit_pages) each. 270 */ 271void balance_dirty_pages_ratelimited_nr(struct address_space *mapping, 272 unsigned long nr_pages_dirtied) 273{ 274 static DEFINE_PER_CPU(unsigned long, ratelimits) = 0; 275 unsigned long ratelimit; 276 unsigned long *p; 277 278 ratelimit = ratelimit_pages; 279 if (dirty_exceeded) 280 ratelimit = 8; 281 282 /* 283 * Check the rate limiting. Also, we do not want to throttle real-time 284 * tasks in balance_dirty_pages(). Period. 285 */ 286 preempt_disable(); 287 p = &__get_cpu_var(ratelimits); 288 *p += nr_pages_dirtied; 289 if (unlikely(*p >= ratelimit)) { 290 *p = 0; 291 preempt_enable(); 292 balance_dirty_pages(mapping); 293 return; 294 } 295 preempt_enable(); 296} 297EXPORT_SYMBOL(balance_dirty_pages_ratelimited_nr); 298 299void throttle_vm_writeout(gfp_t gfp_mask) 300{ 301 long background_thresh; 302 long dirty_thresh; 303 304 if ((gfp_mask & (__GFP_FS|__GFP_IO)) != (__GFP_FS|__GFP_IO)) { 305 /* 306 * The caller might hold locks which can prevent IO completion 307 * or progress in the filesystem. So we cannot just sit here 308 * waiting for IO to complete. 309 */ 310 congestion_wait(WRITE, HZ/10); 311 return; 312 } 313 314 for ( ; ; ) { 315 get_dirty_limits(&background_thresh, &dirty_thresh, NULL); 316 317 /* 318 * Boost the allowable dirty threshold a bit for page 319 * allocators so they don't get DoS'ed by heavy writers 320 */ 321 dirty_thresh += dirty_thresh / 10; /* wheeee... */ 322 323 if (global_page_state(NR_UNSTABLE_NFS) + 324 global_page_state(NR_WRITEBACK) <= dirty_thresh) 325 break; 326 congestion_wait(WRITE, HZ/10); 327 } 328} 329 330/* 331 * writeback at least _min_pages, and keep writing until the amount of dirty 332 * memory is less than the background threshold, or until we're all clean. 333 */ 334static void background_writeout(unsigned long _min_pages) 335{ 336 long min_pages = _min_pages; 337 struct writeback_control wbc = { 338 .bdi = NULL, 339 .sync_mode = WB_SYNC_NONE, 340 .older_than_this = NULL, 341 .nr_to_write = 0, 342 .nonblocking = 1, 343 .range_cyclic = 1, 344 }; 345 346 for ( ; ; ) { 347 long background_thresh; 348 long dirty_thresh; 349 350 get_dirty_limits(&background_thresh, &dirty_thresh, NULL); 351 if (global_page_state(NR_FILE_DIRTY) + 352 global_page_state(NR_UNSTABLE_NFS) < background_thresh 353 && min_pages <= 0) 354 break; 355 wbc.encountered_congestion = 0; 356 wbc.nr_to_write = MAX_WRITEBACK_PAGES; 357 wbc.pages_skipped = 0; 358 writeback_inodes(&wbc); 359 min_pages -= MAX_WRITEBACK_PAGES - wbc.nr_to_write; 360 if (wbc.nr_to_write > 0 || wbc.pages_skipped > 0) { 361 /* Wrote less than expected */ 362 congestion_wait(WRITE, HZ/10); 363 if (!wbc.encountered_congestion) 364 break; 365 } 366 } 367} 368 369/* 370 * Start writeback of `nr_pages' pages. If `nr_pages' is zero, write back 371 * the whole world. Returns 0 if a pdflush thread was dispatched. Returns 372 * -1 if all pdflush threads were busy. 373 */ 374int wakeup_pdflush(long nr_pages) 375{ 376 if (nr_pages == 0) 377 nr_pages = global_page_state(NR_FILE_DIRTY) + 378 global_page_state(NR_UNSTABLE_NFS); 379 return pdflush_operation(background_writeout, nr_pages); 380} 381 382static void wb_timer_fn(unsigned long unused); 383static void laptop_timer_fn(unsigned long unused); 384 385static DEFINE_TIMER(wb_timer, wb_timer_fn, 0, 0); 386static DEFINE_TIMER(laptop_mode_wb_timer, laptop_timer_fn, 0, 0); 387 388/* 389 * Periodic writeback of "old" data. 390 * 391 * Define "old": the first time one of an inode's pages is dirtied, we mark the 392 * dirtying-time in the inode's address_space. So this periodic writeback code 393 * just walks the superblock inode list, writing back any inodes which are 394 * older than a specific point in time. 395 * 396 * Try to run once per dirty_writeback_interval. But if a writeback event 397 * takes longer than a dirty_writeback_interval interval, then leave a 398 * one-second gap. 399 * 400 * older_than_this takes precedence over nr_to_write. So we'll only write back 401 * all dirty pages if they are all attached to "old" mappings. 402 */ 403static void wb_kupdate(unsigned long arg) 404{ 405 unsigned long oldest_jif; 406 unsigned long start_jif; 407 unsigned long next_jif; 408 long nr_to_write; 409 struct writeback_control wbc = { 410 .bdi = NULL, 411 .sync_mode = WB_SYNC_NONE, 412 .older_than_this = &oldest_jif, 413 .nr_to_write = 0, 414 .nonblocking = 1, 415 .for_kupdate = 1, 416 .range_cyclic = 1, 417 }; 418 419 sync_supers(); 420 421 oldest_jif = jiffies - dirty_expire_interval; 422 start_jif = jiffies; 423 next_jif = start_jif + dirty_writeback_interval; 424 nr_to_write = global_page_state(NR_FILE_DIRTY) + 425 global_page_state(NR_UNSTABLE_NFS) + 426 (inodes_stat.nr_inodes - inodes_stat.nr_unused); 427 while (nr_to_write > 0) { 428 wbc.encountered_congestion = 0; 429 wbc.nr_to_write = MAX_WRITEBACK_PAGES; 430 writeback_inodes(&wbc); 431 if (wbc.nr_to_write > 0) { 432 if (wbc.encountered_congestion) 433 congestion_wait(WRITE, HZ/10); 434 else 435 break; /* All the old data is written */ 436 } 437 nr_to_write -= MAX_WRITEBACK_PAGES - wbc.nr_to_write; 438 } 439 if (time_before(next_jif, jiffies + HZ)) 440 next_jif = jiffies + HZ; 441 if (dirty_writeback_interval) 442 mod_timer(&wb_timer, next_jif); 443} 444 445/* 446 * sysctl handler for /proc/sys/vm/dirty_writeback_centisecs 447 */ 448int dirty_writeback_centisecs_handler(ctl_table *table, int write, 449 struct file *file, void __user *buffer, size_t *length, loff_t *ppos) 450{ 451 proc_dointvec_userhz_jiffies(table, write, file, buffer, length, ppos); 452 if (dirty_writeback_interval) { 453 mod_timer(&wb_timer, 454 jiffies + dirty_writeback_interval); 455 } else { 456 del_timer(&wb_timer); 457 } 458 return 0; 459} 460 461static void wb_timer_fn(unsigned long unused) 462{ 463 if (pdflush_operation(wb_kupdate, 0) < 0) 464 mod_timer(&wb_timer, jiffies + HZ); /* delay 1 second */ 465} 466 467static void laptop_flush(unsigned long unused) 468{ 469 sys_sync(); 470} 471 472static void laptop_timer_fn(unsigned long unused) 473{ 474 pdflush_operation(laptop_flush, 0); 475} 476 477/* 478 * We've spun up the disk and we're in laptop mode: schedule writeback 479 * of all dirty data a few seconds from now. If the flush is already scheduled 480 * then push it back - the user is still using the disk. 481 */ 482void laptop_io_completion(void) 483{ 484 mod_timer(&laptop_mode_wb_timer, jiffies + laptop_mode); 485} 486 487/* 488 * We're in laptop mode and we've just synced. The sync's writes will have 489 * caused another writeback to be scheduled by laptop_io_completion. 490 * Nothing needs to be written back anymore, so we unschedule the writeback. 491 */ 492void laptop_sync_completion(void) 493{ 494 del_timer(&laptop_mode_wb_timer); 495} 496 497/* 498 * If ratelimit_pages is too high then we can get into dirty-data overload 499 * if a large number of processes all perform writes at the same time. 500 * If it is too low then SMP machines will call the (expensive) 501 * get_writeback_state too often. 502 * 503 * Here we set ratelimit_pages to a level which ensures that when all CPUs are 504 * dirtying in parallel, we cannot go more than 3% (1/32) over the dirty memory 505 * thresholds before writeback cuts in. 506 * 507 * But the limit should not be set too high. Because it also controls the 508 * amount of memory which the balance_dirty_pages() caller has to write back. 509 * If this is too large then the caller will block on the IO queue all the 510 * time. So limit it to four megabytes - the balance_dirty_pages() caller 511 * will write six megabyte chunks, max. 512 */ 513 514void writeback_set_ratelimit(void) 515{ 516 ratelimit_pages = vm_total_pages / (num_online_cpus() * 32); 517 if (ratelimit_pages < 16) 518 ratelimit_pages = 16; 519 if (ratelimit_pages * PAGE_CACHE_SIZE > 4096 * 1024) 520 ratelimit_pages = (4096 * 1024) / PAGE_CACHE_SIZE; 521} 522 523static int __cpuinit 524ratelimit_handler(struct notifier_block *self, unsigned long u, void *v) 525{ 526 writeback_set_ratelimit(); 527 return NOTIFY_DONE; 528} 529 530static struct notifier_block __cpuinitdata ratelimit_nb = { 531 .notifier_call = ratelimit_handler, 532 .next = NULL, 533}; 534 535/* 536 * Called early on to tune the page writeback dirty limits. 537 * 538 * We used to scale dirty pages according to how total memory 539 * related to pages that could be allocated for buffers (by 540 * comparing nr_free_buffer_pages() to vm_total_pages. 541 * 542 * However, that was when we used "dirty_ratio" to scale with 543 * all memory, and we don't do that any more. "dirty_ratio" 544 * is now applied to total non-HIGHPAGE memory (by subtracting 545 * totalhigh_pages from vm_total_pages), and as such we can't 546 * get into the old insane situation any more where we had 547 * large amounts of dirty pages compared to a small amount of 548 * non-HIGHMEM memory. 549 * 550 * But we might still want to scale the dirty_ratio by how 551 * much memory the box has.. 552 */ 553void __init page_writeback_init(void) 554{ 555 mod_timer(&wb_timer, jiffies + dirty_writeback_interval); 556 writeback_set_ratelimit(); 557 register_cpu_notifier(&ratelimit_nb); 558} 559 560/** 561 * generic_writepages - walk the list of dirty pages of the given address space and writepage() all of them. 562 * @mapping: address space structure to write 563 * @wbc: subtract the number of written pages from *@wbc->nr_to_write 564 * 565 * This is a library function, which implements the writepages() 566 * address_space_operation. 567 * 568 * If a page is already under I/O, generic_writepages() skips it, even 569 * if it's dirty. This is desirable behaviour for memory-cleaning writeback, 570 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync() 571 * and msync() need to guarantee that all the data which was dirty at the time 572 * the call was made get new I/O started against them. If wbc->sync_mode is 573 * WB_SYNC_ALL then we were called for data integrity and we must wait for 574 * existing IO to complete. 575 * 576 * Derived from mpage_writepages() - if you fix this you should check that 577 * also! 578 */ 579int generic_writepages(struct address_space *mapping, 580 struct writeback_control *wbc) 581{ 582 struct backing_dev_info *bdi = mapping->backing_dev_info; 583 int ret = 0; 584 int done = 0; 585 int (*writepage)(struct page *page, struct writeback_control *wbc); 586 struct pagevec pvec; 587 int nr_pages; 588 pgoff_t index; 589 pgoff_t end; /* Inclusive */ 590 int scanned = 0; 591 int range_whole = 0; 592 593 if (wbc->nonblocking && bdi_write_congested(bdi)) { 594 wbc->encountered_congestion = 1; 595 return 0; 596 } 597 598 writepage = mapping->a_ops->writepage; 599 600 /* deal with chardevs and other special file */ 601 if (!writepage) 602 return 0; 603 604 pagevec_init(&pvec, 0); 605 if (wbc->range_cyclic) { 606 index = mapping->writeback_index; /* Start from prev offset */ 607 end = -1; 608 } else { 609 index = wbc->range_start >> PAGE_CACHE_SHIFT; 610 end = wbc->range_end >> PAGE_CACHE_SHIFT; 611 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 612 range_whole = 1; 613 scanned = 1; 614 } 615retry: 616 while (!done && (index <= end) && 617 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, 618 PAGECACHE_TAG_DIRTY, 619 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) { 620 unsigned i; 621 622 scanned = 1; 623 for (i = 0; i < nr_pages; i++) { 624 struct page *page = pvec.pages[i]; 625 626 /* 627 * At this point we hold neither mapping->tree_lock nor 628 * lock on the page itself: the page may be truncated or 629 * invalidated (changing page->mapping to NULL), or even 630 * swizzled back from swapper_space to tmpfs file 631 * mapping 632 */ 633 lock_page(page); 634 635 if (unlikely(page->mapping != mapping)) { 636 unlock_page(page); 637 continue; 638 } 639 640 if (!wbc->range_cyclic && page->index > end) { 641 done = 1; 642 unlock_page(page); 643 continue; 644 } 645 646 if (wbc->sync_mode != WB_SYNC_NONE) 647 wait_on_page_writeback(page); 648 649 if (PageWriteback(page) || 650 !clear_page_dirty_for_io(page)) { 651 unlock_page(page); 652 continue; 653 } 654 655 ret = (*writepage)(page, wbc); 656 if (ret) { 657 if (ret == -ENOSPC) 658 set_bit(AS_ENOSPC, &mapping->flags); 659 else 660 set_bit(AS_EIO, &mapping->flags); 661 } 662 663 if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) 664 unlock_page(page); 665 if (ret || (--(wbc->nr_to_write) <= 0)) 666 done = 1; 667 if (wbc->nonblocking && bdi_write_congested(bdi)) { 668 wbc->encountered_congestion = 1; 669 done = 1; 670 } 671 } 672 pagevec_release(&pvec); 673 cond_resched(); 674 } 675 if (!scanned && !done) { 676 /* 677 * We hit the last page and there is more work to be done: wrap 678 * back to the start of the file 679 */ 680 scanned = 1; 681 index = 0; 682 goto retry; 683 } 684 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) 685 mapping->writeback_index = index; 686 return ret; 687} 688 689EXPORT_SYMBOL(generic_writepages); 690 691int do_writepages(struct address_space *mapping, struct writeback_control *wbc) 692{ 693 int ret; 694 695 if (wbc->nr_to_write <= 0) 696 return 0; 697 wbc->for_writepages = 1; 698 if (mapping->a_ops->writepages) 699 ret = mapping->a_ops->writepages(mapping, wbc); 700 else 701 ret = generic_writepages(mapping, wbc); 702 wbc->for_writepages = 0; 703 return ret; 704} 705 706/** 707 * write_one_page - write out a single page and optionally wait on I/O 708 * @page: the page to write 709 * @wait: if true, wait on writeout 710 * 711 * The page must be locked by the caller and will be unlocked upon return. 712 * 713 * write_one_page() returns a negative error code if I/O failed. 714 */ 715int write_one_page(struct page *page, int wait) 716{ 717 struct address_space *mapping = page->mapping; 718 int ret = 0; 719 struct writeback_control wbc = { 720 .sync_mode = WB_SYNC_ALL, 721 .nr_to_write = 1, 722 }; 723 724 BUG_ON(!PageLocked(page)); 725 726 if (wait) 727 wait_on_page_writeback(page); 728 729 if (clear_page_dirty_for_io(page)) { 730 page_cache_get(page); 731 ret = mapping->a_ops->writepage(page, &wbc); 732 if (ret == 0 && wait) { 733 wait_on_page_writeback(page); 734 if (PageError(page)) 735 ret = -EIO; 736 } 737 page_cache_release(page); 738 } else { 739 unlock_page(page); 740 } 741 return ret; 742} 743EXPORT_SYMBOL(write_one_page); 744 745/* 746 * For address_spaces which do not use buffers nor write back. 747 */ 748int __set_page_dirty_no_writeback(struct page *page) 749{ 750 if (!PageDirty(page)) 751 SetPageDirty(page); 752 return 0; 753} 754 755/* 756 * For address_spaces which do not use buffers. Just tag the page as dirty in 757 * its radix tree. 758 * 759 * This is also used when a single buffer is being dirtied: we want to set the 760 * page dirty in that case, but not all the buffers. This is a "bottom-up" 761 * dirtying, whereas __set_page_dirty_buffers() is a "top-down" dirtying. 762 * 763 * Most callers have locked the page, which pins the address_space in memory. 764 * But zap_pte_range() does not lock the page, however in that case the 765 * mapping is pinned by the vma's ->vm_file reference. 766 * 767 * We take care to handle the case where the page was truncated from the 768 * mapping by re-checking page_mapping() insode tree_lock. 769 */ 770int __set_page_dirty_nobuffers(struct page *page) 771{ 772 if (!TestSetPageDirty(page)) { 773 struct address_space *mapping = page_mapping(page); 774 struct address_space *mapping2; 775 776 if (!mapping) 777 return 1; 778 779 write_lock_irq(&mapping->tree_lock); 780 mapping2 = page_mapping(page); 781 if (mapping2) { /* Race with truncate? */ 782 BUG_ON(mapping2 != mapping); 783 if (mapping_cap_account_dirty(mapping)) { 784 __inc_zone_page_state(page, NR_FILE_DIRTY); 785 task_io_account_write(PAGE_CACHE_SIZE); 786 } 787 radix_tree_tag_set(&mapping->page_tree, 788 page_index(page), PAGECACHE_TAG_DIRTY); 789 } 790 write_unlock_irq(&mapping->tree_lock); 791 if (mapping->host) { 792 /* !PageAnon && !swapper_space */ 793 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); 794 } 795 return 1; 796 } 797 return 0; 798} 799EXPORT_SYMBOL(__set_page_dirty_nobuffers); 800 801/* 802 * When a writepage implementation decides that it doesn't want to write this 803 * page for some reason, it should redirty the locked page via 804 * redirty_page_for_writepage() and it should then unlock the page and return 0 805 */ 806int redirty_page_for_writepage(struct writeback_control *wbc, struct page *page) 807{ 808 wbc->pages_skipped++; 809 return __set_page_dirty_nobuffers(page); 810} 811EXPORT_SYMBOL(redirty_page_for_writepage); 812 813/* 814 * If the mapping doesn't provide a set_page_dirty a_op, then 815 * just fall through and assume that it wants buffer_heads. 816 */ 817int fastcall set_page_dirty(struct page *page) 818{ 819 struct address_space *mapping = page_mapping(page); 820 821 if (likely(mapping)) { 822 int (*spd)(struct page *) = mapping->a_ops->set_page_dirty; 823#ifdef CONFIG_BLOCK 824 if (!spd) 825 spd = __set_page_dirty_buffers; 826#endif 827 return (*spd)(page); 828 } 829 if (!PageDirty(page)) { 830 if (!TestSetPageDirty(page)) 831 return 1; 832 } 833 return 0; 834} 835EXPORT_SYMBOL(set_page_dirty); 836 837/* 838 * set_page_dirty() is racy if the caller has no reference against 839 * page->mapping->host, and if the page is unlocked. This is because another 840 * CPU could truncate the page off the mapping and then free the mapping. 841 * 842 * Usually, the page _is_ locked, or the caller is a user-space process which 843 * holds a reference on the inode by having an open file. 844 * 845 * In other cases, the page should be locked before running set_page_dirty(). 846 */ 847int set_page_dirty_lock(struct page *page) 848{ 849 int ret; 850 851 lock_page_nosync(page); 852 ret = set_page_dirty(page); 853 unlock_page(page); 854 return ret; 855} 856EXPORT_SYMBOL(set_page_dirty_lock); 857 858/* 859 * Clear a page's dirty flag, while caring for dirty memory accounting. 860 * Returns true if the page was previously dirty. 861 * 862 * This is for preparing to put the page under writeout. We leave the page 863 * tagged as dirty in the radix tree so that a concurrent write-for-sync 864 * can discover it via a PAGECACHE_TAG_DIRTY walk. The ->writepage 865 * implementation will run either set_page_writeback() or set_page_dirty(), 866 * at which stage we bring the page's dirty flag and radix-tree dirty tag 867 * back into sync. 868 * 869 * This incoherency between the page's dirty flag and radix-tree tag is 870 * unfortunate, but it only exists while the page is locked. 871 */ 872int clear_page_dirty_for_io(struct page *page) 873{ 874 struct address_space *mapping = page_mapping(page); 875 876 if (mapping && mapping_cap_account_dirty(mapping)) { 877 /* 878 * Yes, Virginia, this is indeed insane. 879 * 880 * We use this sequence to make sure that 881 * (a) we account for dirty stats properly 882 * (b) we tell the low-level filesystem to 883 * mark the whole page dirty if it was 884 * dirty in a pagetable. Only to then 885 * (c) clean the page again and return 1 to 886 * cause the writeback. 887 * 888 * This way we avoid all nasty races with the 889 * dirty bit in multiple places and clearing 890 * them concurrently from different threads. 891 * 892 * Note! Normally the "set_page_dirty(page)" 893 * has no effect on the actual dirty bit - since 894 * that will already usually be set. But we 895 * need the side effects, and it can help us 896 * avoid races. 897 * 898 * We basically use the page "master dirty bit" 899 * as a serialization point for all the different 900 * threads doing their things. 901 * 902 * FIXME! We still have a race here: if somebody 903 * adds the page back to the page tables in 904 * between the "page_mkclean()" and the "TestClearPageDirty()", 905 * we might have it mapped without the dirty bit set. 906 */ 907 if (page_mkclean(page)) 908 set_page_dirty(page); 909 if (TestClearPageDirty(page)) { 910 dec_zone_page_state(page, NR_FILE_DIRTY); 911 return 1; 912 } 913 return 0; 914 } 915 return TestClearPageDirty(page); 916} 917EXPORT_SYMBOL(clear_page_dirty_for_io); 918 919int test_clear_page_writeback(struct page *page) 920{ 921 struct address_space *mapping = page_mapping(page); 922 int ret; 923 924 if (mapping) { 925 unsigned long flags; 926 927 write_lock_irqsave(&mapping->tree_lock, flags); 928 ret = TestClearPageWriteback(page); 929 if (ret) 930 radix_tree_tag_clear(&mapping->page_tree, 931 page_index(page), 932 PAGECACHE_TAG_WRITEBACK); 933 write_unlock_irqrestore(&mapping->tree_lock, flags); 934 } else { 935 ret = TestClearPageWriteback(page); 936 } 937 return ret; 938} 939 940int test_set_page_writeback(struct page *page) 941{ 942 struct address_space *mapping = page_mapping(page); 943 int ret; 944 945 if (mapping) { 946 unsigned long flags; 947 948 write_lock_irqsave(&mapping->tree_lock, flags); 949 ret = TestSetPageWriteback(page); 950 if (!ret) 951 radix_tree_tag_set(&mapping->page_tree, 952 page_index(page), 953 PAGECACHE_TAG_WRITEBACK); 954 if (!PageDirty(page)) 955 radix_tree_tag_clear(&mapping->page_tree, 956 page_index(page), 957 PAGECACHE_TAG_DIRTY); 958 write_unlock_irqrestore(&mapping->tree_lock, flags); 959 } else { 960 ret = TestSetPageWriteback(page); 961 } 962 return ret; 963 964} 965EXPORT_SYMBOL(test_set_page_writeback); 966 967/* 968 * Return true if any of the pages in the mapping are marged with the 969 * passed tag. 970 */ 971int mapping_tagged(struct address_space *mapping, int tag) 972{ 973 unsigned long flags; 974 int ret; 975 976 read_lock_irqsave(&mapping->tree_lock, flags); 977 ret = radix_tree_tagged(&mapping->page_tree, tag); 978 read_unlock_irqrestore(&mapping->tree_lock, flags); 979 return ret; 980} 981EXPORT_SYMBOL(mapping_tagged); 982