filemap.c revision c0ff7453bb5c7c98e0885fb94279f2571946f280
1/* 2 * linux/mm/filemap.c 3 * 4 * Copyright (C) 1994-1999 Linus Torvalds 5 */ 6 7/* 8 * This file handles the generic file mmap semantics used by 9 * most "normal" filesystems (but you don't /have/ to use this: 10 * the NFS filesystem used to do this differently, for example) 11 */ 12#include <linux/module.h> 13#include <linux/compiler.h> 14#include <linux/fs.h> 15#include <linux/uaccess.h> 16#include <linux/aio.h> 17#include <linux/capability.h> 18#include <linux/kernel_stat.h> 19#include <linux/gfp.h> 20#include <linux/mm.h> 21#include <linux/swap.h> 22#include <linux/mman.h> 23#include <linux/pagemap.h> 24#include <linux/file.h> 25#include <linux/uio.h> 26#include <linux/hash.h> 27#include <linux/writeback.h> 28#include <linux/backing-dev.h> 29#include <linux/pagevec.h> 30#include <linux/blkdev.h> 31#include <linux/security.h> 32#include <linux/syscalls.h> 33#include <linux/cpuset.h> 34#include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */ 35#include <linux/memcontrol.h> 36#include <linux/mm_inline.h> /* for page_is_file_cache() */ 37#include "internal.h" 38 39/* 40 * FIXME: remove all knowledge of the buffer layer from the core VM 41 */ 42#include <linux/buffer_head.h> /* for try_to_free_buffers */ 43 44#include <asm/mman.h> 45 46/* 47 * Shared mappings implemented 30.11.1994. It's not fully working yet, 48 * though. 49 * 50 * Shared mappings now work. 15.8.1995 Bruno. 51 * 52 * finished 'unifying' the page and buffer cache and SMP-threaded the 53 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com> 54 * 55 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de> 56 */ 57 58/* 59 * Lock ordering: 60 * 61 * ->i_mmap_lock (truncate_pagecache) 62 * ->private_lock (__free_pte->__set_page_dirty_buffers) 63 * ->swap_lock (exclusive_swap_page, others) 64 * ->mapping->tree_lock 65 * 66 * ->i_mutex 67 * ->i_mmap_lock (truncate->unmap_mapping_range) 68 * 69 * ->mmap_sem 70 * ->i_mmap_lock 71 * ->page_table_lock or pte_lock (various, mainly in memory.c) 72 * ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock) 73 * 74 * ->mmap_sem 75 * ->lock_page (access_process_vm) 76 * 77 * ->i_mutex (generic_file_buffered_write) 78 * ->mmap_sem (fault_in_pages_readable->do_page_fault) 79 * 80 * ->i_mutex 81 * ->i_alloc_sem (various) 82 * 83 * ->inode_lock 84 * ->sb_lock (fs/fs-writeback.c) 85 * ->mapping->tree_lock (__sync_single_inode) 86 * 87 * ->i_mmap_lock 88 * ->anon_vma.lock (vma_adjust) 89 * 90 * ->anon_vma.lock 91 * ->page_table_lock or pte_lock (anon_vma_prepare and various) 92 * 93 * ->page_table_lock or pte_lock 94 * ->swap_lock (try_to_unmap_one) 95 * ->private_lock (try_to_unmap_one) 96 * ->tree_lock (try_to_unmap_one) 97 * ->zone.lru_lock (follow_page->mark_page_accessed) 98 * ->zone.lru_lock (check_pte_range->isolate_lru_page) 99 * ->private_lock (page_remove_rmap->set_page_dirty) 100 * ->tree_lock (page_remove_rmap->set_page_dirty) 101 * ->inode_lock (page_remove_rmap->set_page_dirty) 102 * ->inode_lock (zap_pte_range->set_page_dirty) 103 * ->private_lock (zap_pte_range->__set_page_dirty_buffers) 104 * 105 * ->task->proc_lock 106 * ->dcache_lock (proc_pid_lookup) 107 * 108 * (code doesn't rely on that order, so you could switch it around) 109 * ->tasklist_lock (memory_failure, collect_procs_ao) 110 * ->i_mmap_lock 111 */ 112 113/* 114 * Remove a page from the page cache and free it. Caller has to make 115 * sure the page is locked and that nobody else uses it - or that usage 116 * is safe. The caller must hold the mapping's tree_lock. 117 */ 118void __remove_from_page_cache(struct page *page) 119{ 120 struct address_space *mapping = page->mapping; 121 122 radix_tree_delete(&mapping->page_tree, page->index); 123 page->mapping = NULL; 124 mapping->nrpages--; 125 __dec_zone_page_state(page, NR_FILE_PAGES); 126 if (PageSwapBacked(page)) 127 __dec_zone_page_state(page, NR_SHMEM); 128 BUG_ON(page_mapped(page)); 129 130 /* 131 * Some filesystems seem to re-dirty the page even after 132 * the VM has canceled the dirty bit (eg ext3 journaling). 133 * 134 * Fix it up by doing a final dirty accounting check after 135 * having removed the page entirely. 136 */ 137 if (PageDirty(page) && mapping_cap_account_dirty(mapping)) { 138 dec_zone_page_state(page, NR_FILE_DIRTY); 139 dec_bdi_stat(mapping->backing_dev_info, BDI_RECLAIMABLE); 140 } 141} 142 143void remove_from_page_cache(struct page *page) 144{ 145 struct address_space *mapping = page->mapping; 146 147 BUG_ON(!PageLocked(page)); 148 149 spin_lock_irq(&mapping->tree_lock); 150 __remove_from_page_cache(page); 151 spin_unlock_irq(&mapping->tree_lock); 152 mem_cgroup_uncharge_cache_page(page); 153} 154 155static int sync_page(void *word) 156{ 157 struct address_space *mapping; 158 struct page *page; 159 160 page = container_of((unsigned long *)word, struct page, flags); 161 162 /* 163 * page_mapping() is being called without PG_locked held. 164 * Some knowledge of the state and use of the page is used to 165 * reduce the requirements down to a memory barrier. 166 * The danger here is of a stale page_mapping() return value 167 * indicating a struct address_space different from the one it's 168 * associated with when it is associated with one. 169 * After smp_mb(), it's either the correct page_mapping() for 170 * the page, or an old page_mapping() and the page's own 171 * page_mapping() has gone NULL. 172 * The ->sync_page() address_space operation must tolerate 173 * page_mapping() going NULL. By an amazing coincidence, 174 * this comes about because none of the users of the page 175 * in the ->sync_page() methods make essential use of the 176 * page_mapping(), merely passing the page down to the backing 177 * device's unplug functions when it's non-NULL, which in turn 178 * ignore it for all cases but swap, where only page_private(page) is 179 * of interest. When page_mapping() does go NULL, the entire 180 * call stack gracefully ignores the page and returns. 181 * -- wli 182 */ 183 smp_mb(); 184 mapping = page_mapping(page); 185 if (mapping && mapping->a_ops && mapping->a_ops->sync_page) 186 mapping->a_ops->sync_page(page); 187 io_schedule(); 188 return 0; 189} 190 191static int sync_page_killable(void *word) 192{ 193 sync_page(word); 194 return fatal_signal_pending(current) ? -EINTR : 0; 195} 196 197/** 198 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range 199 * @mapping: address space structure to write 200 * @start: offset in bytes where the range starts 201 * @end: offset in bytes where the range ends (inclusive) 202 * @sync_mode: enable synchronous operation 203 * 204 * Start writeback against all of a mapping's dirty pages that lie 205 * within the byte offsets <start, end> inclusive. 206 * 207 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as 208 * opposed to a regular memory cleansing writeback. The difference between 209 * these two operations is that if a dirty page/buffer is encountered, it must 210 * be waited upon, and not just skipped over. 211 */ 212int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start, 213 loff_t end, int sync_mode) 214{ 215 int ret; 216 struct writeback_control wbc = { 217 .sync_mode = sync_mode, 218 .nr_to_write = LONG_MAX, 219 .range_start = start, 220 .range_end = end, 221 }; 222 223 if (!mapping_cap_writeback_dirty(mapping)) 224 return 0; 225 226 ret = do_writepages(mapping, &wbc); 227 return ret; 228} 229 230static inline int __filemap_fdatawrite(struct address_space *mapping, 231 int sync_mode) 232{ 233 return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode); 234} 235 236int filemap_fdatawrite(struct address_space *mapping) 237{ 238 return __filemap_fdatawrite(mapping, WB_SYNC_ALL); 239} 240EXPORT_SYMBOL(filemap_fdatawrite); 241 242int filemap_fdatawrite_range(struct address_space *mapping, loff_t start, 243 loff_t end) 244{ 245 return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL); 246} 247EXPORT_SYMBOL(filemap_fdatawrite_range); 248 249/** 250 * filemap_flush - mostly a non-blocking flush 251 * @mapping: target address_space 252 * 253 * This is a mostly non-blocking flush. Not suitable for data-integrity 254 * purposes - I/O may not be started against all dirty pages. 255 */ 256int filemap_flush(struct address_space *mapping) 257{ 258 return __filemap_fdatawrite(mapping, WB_SYNC_NONE); 259} 260EXPORT_SYMBOL(filemap_flush); 261 262/** 263 * filemap_fdatawait_range - wait for writeback to complete 264 * @mapping: address space structure to wait for 265 * @start_byte: offset in bytes where the range starts 266 * @end_byte: offset in bytes where the range ends (inclusive) 267 * 268 * Walk the list of under-writeback pages of the given address space 269 * in the given range and wait for all of them. 270 */ 271int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte, 272 loff_t end_byte) 273{ 274 pgoff_t index = start_byte >> PAGE_CACHE_SHIFT; 275 pgoff_t end = end_byte >> PAGE_CACHE_SHIFT; 276 struct pagevec pvec; 277 int nr_pages; 278 int ret = 0; 279 280 if (end_byte < start_byte) 281 return 0; 282 283 pagevec_init(&pvec, 0); 284 while ((index <= end) && 285 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, 286 PAGECACHE_TAG_WRITEBACK, 287 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1)) != 0) { 288 unsigned i; 289 290 for (i = 0; i < nr_pages; i++) { 291 struct page *page = pvec.pages[i]; 292 293 /* until radix tree lookup accepts end_index */ 294 if (page->index > end) 295 continue; 296 297 wait_on_page_writeback(page); 298 if (PageError(page)) 299 ret = -EIO; 300 } 301 pagevec_release(&pvec); 302 cond_resched(); 303 } 304 305 /* Check for outstanding write errors */ 306 if (test_and_clear_bit(AS_ENOSPC, &mapping->flags)) 307 ret = -ENOSPC; 308 if (test_and_clear_bit(AS_EIO, &mapping->flags)) 309 ret = -EIO; 310 311 return ret; 312} 313EXPORT_SYMBOL(filemap_fdatawait_range); 314 315/** 316 * filemap_fdatawait - wait for all under-writeback pages to complete 317 * @mapping: address space structure to wait for 318 * 319 * Walk the list of under-writeback pages of the given address space 320 * and wait for all of them. 321 */ 322int filemap_fdatawait(struct address_space *mapping) 323{ 324 loff_t i_size = i_size_read(mapping->host); 325 326 if (i_size == 0) 327 return 0; 328 329 return filemap_fdatawait_range(mapping, 0, i_size - 1); 330} 331EXPORT_SYMBOL(filemap_fdatawait); 332 333int filemap_write_and_wait(struct address_space *mapping) 334{ 335 int err = 0; 336 337 if (mapping->nrpages) { 338 err = filemap_fdatawrite(mapping); 339 /* 340 * Even if the above returned error, the pages may be 341 * written partially (e.g. -ENOSPC), so we wait for it. 342 * But the -EIO is special case, it may indicate the worst 343 * thing (e.g. bug) happened, so we avoid waiting for it. 344 */ 345 if (err != -EIO) { 346 int err2 = filemap_fdatawait(mapping); 347 if (!err) 348 err = err2; 349 } 350 } 351 return err; 352} 353EXPORT_SYMBOL(filemap_write_and_wait); 354 355/** 356 * filemap_write_and_wait_range - write out & wait on a file range 357 * @mapping: the address_space for the pages 358 * @lstart: offset in bytes where the range starts 359 * @lend: offset in bytes where the range ends (inclusive) 360 * 361 * Write out and wait upon file offsets lstart->lend, inclusive. 362 * 363 * Note that `lend' is inclusive (describes the last byte to be written) so 364 * that this function can be used to write to the very end-of-file (end = -1). 365 */ 366int filemap_write_and_wait_range(struct address_space *mapping, 367 loff_t lstart, loff_t lend) 368{ 369 int err = 0; 370 371 if (mapping->nrpages) { 372 err = __filemap_fdatawrite_range(mapping, lstart, lend, 373 WB_SYNC_ALL); 374 /* See comment of filemap_write_and_wait() */ 375 if (err != -EIO) { 376 int err2 = filemap_fdatawait_range(mapping, 377 lstart, lend); 378 if (!err) 379 err = err2; 380 } 381 } 382 return err; 383} 384EXPORT_SYMBOL(filemap_write_and_wait_range); 385 386/** 387 * add_to_page_cache_locked - add a locked page to the pagecache 388 * @page: page to add 389 * @mapping: the page's address_space 390 * @offset: page index 391 * @gfp_mask: page allocation mode 392 * 393 * This function is used to add a page to the pagecache. It must be locked. 394 * This function does not add the page to the LRU. The caller must do that. 395 */ 396int add_to_page_cache_locked(struct page *page, struct address_space *mapping, 397 pgoff_t offset, gfp_t gfp_mask) 398{ 399 int error; 400 401 VM_BUG_ON(!PageLocked(page)); 402 403 error = mem_cgroup_cache_charge(page, current->mm, 404 gfp_mask & GFP_RECLAIM_MASK); 405 if (error) 406 goto out; 407 408 error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM); 409 if (error == 0) { 410 page_cache_get(page); 411 page->mapping = mapping; 412 page->index = offset; 413 414 spin_lock_irq(&mapping->tree_lock); 415 error = radix_tree_insert(&mapping->page_tree, offset, page); 416 if (likely(!error)) { 417 mapping->nrpages++; 418 __inc_zone_page_state(page, NR_FILE_PAGES); 419 if (PageSwapBacked(page)) 420 __inc_zone_page_state(page, NR_SHMEM); 421 spin_unlock_irq(&mapping->tree_lock); 422 } else { 423 page->mapping = NULL; 424 spin_unlock_irq(&mapping->tree_lock); 425 mem_cgroup_uncharge_cache_page(page); 426 page_cache_release(page); 427 } 428 radix_tree_preload_end(); 429 } else 430 mem_cgroup_uncharge_cache_page(page); 431out: 432 return error; 433} 434EXPORT_SYMBOL(add_to_page_cache_locked); 435 436int add_to_page_cache_lru(struct page *page, struct address_space *mapping, 437 pgoff_t offset, gfp_t gfp_mask) 438{ 439 int ret; 440 441 /* 442 * Splice_read and readahead add shmem/tmpfs pages into the page cache 443 * before shmem_readpage has a chance to mark them as SwapBacked: they 444 * need to go on the anon lru below, and mem_cgroup_cache_charge 445 * (called in add_to_page_cache) needs to know where they're going too. 446 */ 447 if (mapping_cap_swap_backed(mapping)) 448 SetPageSwapBacked(page); 449 450 ret = add_to_page_cache(page, mapping, offset, gfp_mask); 451 if (ret == 0) { 452 if (page_is_file_cache(page)) 453 lru_cache_add_file(page); 454 else 455 lru_cache_add_anon(page); 456 } 457 return ret; 458} 459EXPORT_SYMBOL_GPL(add_to_page_cache_lru); 460 461#ifdef CONFIG_NUMA 462struct page *__page_cache_alloc(gfp_t gfp) 463{ 464 int n; 465 struct page *page; 466 467 if (cpuset_do_page_mem_spread()) { 468 get_mems_allowed(); 469 n = cpuset_mem_spread_node(); 470 page = alloc_pages_exact_node(n, gfp, 0); 471 put_mems_allowed(); 472 return page; 473 } 474 return alloc_pages(gfp, 0); 475} 476EXPORT_SYMBOL(__page_cache_alloc); 477#endif 478 479static int __sleep_on_page_lock(void *word) 480{ 481 io_schedule(); 482 return 0; 483} 484 485/* 486 * In order to wait for pages to become available there must be 487 * waitqueues associated with pages. By using a hash table of 488 * waitqueues where the bucket discipline is to maintain all 489 * waiters on the same queue and wake all when any of the pages 490 * become available, and for the woken contexts to check to be 491 * sure the appropriate page became available, this saves space 492 * at a cost of "thundering herd" phenomena during rare hash 493 * collisions. 494 */ 495static wait_queue_head_t *page_waitqueue(struct page *page) 496{ 497 const struct zone *zone = page_zone(page); 498 499 return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)]; 500} 501 502static inline void wake_up_page(struct page *page, int bit) 503{ 504 __wake_up_bit(page_waitqueue(page), &page->flags, bit); 505} 506 507void wait_on_page_bit(struct page *page, int bit_nr) 508{ 509 DEFINE_WAIT_BIT(wait, &page->flags, bit_nr); 510 511 if (test_bit(bit_nr, &page->flags)) 512 __wait_on_bit(page_waitqueue(page), &wait, sync_page, 513 TASK_UNINTERRUPTIBLE); 514} 515EXPORT_SYMBOL(wait_on_page_bit); 516 517/** 518 * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue 519 * @page: Page defining the wait queue of interest 520 * @waiter: Waiter to add to the queue 521 * 522 * Add an arbitrary @waiter to the wait queue for the nominated @page. 523 */ 524void add_page_wait_queue(struct page *page, wait_queue_t *waiter) 525{ 526 wait_queue_head_t *q = page_waitqueue(page); 527 unsigned long flags; 528 529 spin_lock_irqsave(&q->lock, flags); 530 __add_wait_queue(q, waiter); 531 spin_unlock_irqrestore(&q->lock, flags); 532} 533EXPORT_SYMBOL_GPL(add_page_wait_queue); 534 535/** 536 * unlock_page - unlock a locked page 537 * @page: the page 538 * 539 * Unlocks the page and wakes up sleepers in ___wait_on_page_locked(). 540 * Also wakes sleepers in wait_on_page_writeback() because the wakeup 541 * mechananism between PageLocked pages and PageWriteback pages is shared. 542 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep. 543 * 544 * The mb is necessary to enforce ordering between the clear_bit and the read 545 * of the waitqueue (to avoid SMP races with a parallel wait_on_page_locked()). 546 */ 547void unlock_page(struct page *page) 548{ 549 VM_BUG_ON(!PageLocked(page)); 550 clear_bit_unlock(PG_locked, &page->flags); 551 smp_mb__after_clear_bit(); 552 wake_up_page(page, PG_locked); 553} 554EXPORT_SYMBOL(unlock_page); 555 556/** 557 * end_page_writeback - end writeback against a page 558 * @page: the page 559 */ 560void end_page_writeback(struct page *page) 561{ 562 if (TestClearPageReclaim(page)) 563 rotate_reclaimable_page(page); 564 565 if (!test_clear_page_writeback(page)) 566 BUG(); 567 568 smp_mb__after_clear_bit(); 569 wake_up_page(page, PG_writeback); 570} 571EXPORT_SYMBOL(end_page_writeback); 572 573/** 574 * __lock_page - get a lock on the page, assuming we need to sleep to get it 575 * @page: the page to lock 576 * 577 * Ugly. Running sync_page() in state TASK_UNINTERRUPTIBLE is scary. If some 578 * random driver's requestfn sets TASK_RUNNING, we could busywait. However 579 * chances are that on the second loop, the block layer's plug list is empty, 580 * so sync_page() will then return in state TASK_UNINTERRUPTIBLE. 581 */ 582void __lock_page(struct page *page) 583{ 584 DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); 585 586 __wait_on_bit_lock(page_waitqueue(page), &wait, sync_page, 587 TASK_UNINTERRUPTIBLE); 588} 589EXPORT_SYMBOL(__lock_page); 590 591int __lock_page_killable(struct page *page) 592{ 593 DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); 594 595 return __wait_on_bit_lock(page_waitqueue(page), &wait, 596 sync_page_killable, TASK_KILLABLE); 597} 598EXPORT_SYMBOL_GPL(__lock_page_killable); 599 600/** 601 * __lock_page_nosync - get a lock on the page, without calling sync_page() 602 * @page: the page to lock 603 * 604 * Variant of lock_page that does not require the caller to hold a reference 605 * on the page's mapping. 606 */ 607void __lock_page_nosync(struct page *page) 608{ 609 DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); 610 __wait_on_bit_lock(page_waitqueue(page), &wait, __sleep_on_page_lock, 611 TASK_UNINTERRUPTIBLE); 612} 613 614/** 615 * find_get_page - find and get a page reference 616 * @mapping: the address_space to search 617 * @offset: the page index 618 * 619 * Is there a pagecache struct page at the given (mapping, offset) tuple? 620 * If yes, increment its refcount and return it; if no, return NULL. 621 */ 622struct page *find_get_page(struct address_space *mapping, pgoff_t offset) 623{ 624 void **pagep; 625 struct page *page; 626 627 rcu_read_lock(); 628repeat: 629 page = NULL; 630 pagep = radix_tree_lookup_slot(&mapping->page_tree, offset); 631 if (pagep) { 632 page = radix_tree_deref_slot(pagep); 633 if (unlikely(!page || page == RADIX_TREE_RETRY)) 634 goto repeat; 635 636 if (!page_cache_get_speculative(page)) 637 goto repeat; 638 639 /* 640 * Has the page moved? 641 * This is part of the lockless pagecache protocol. See 642 * include/linux/pagemap.h for details. 643 */ 644 if (unlikely(page != *pagep)) { 645 page_cache_release(page); 646 goto repeat; 647 } 648 } 649 rcu_read_unlock(); 650 651 return page; 652} 653EXPORT_SYMBOL(find_get_page); 654 655/** 656 * find_lock_page - locate, pin and lock a pagecache page 657 * @mapping: the address_space to search 658 * @offset: the page index 659 * 660 * Locates the desired pagecache page, locks it, increments its reference 661 * count and returns its address. 662 * 663 * Returns zero if the page was not present. find_lock_page() may sleep. 664 */ 665struct page *find_lock_page(struct address_space *mapping, pgoff_t offset) 666{ 667 struct page *page; 668 669repeat: 670 page = find_get_page(mapping, offset); 671 if (page) { 672 lock_page(page); 673 /* Has the page been truncated? */ 674 if (unlikely(page->mapping != mapping)) { 675 unlock_page(page); 676 page_cache_release(page); 677 goto repeat; 678 } 679 VM_BUG_ON(page->index != offset); 680 } 681 return page; 682} 683EXPORT_SYMBOL(find_lock_page); 684 685/** 686 * find_or_create_page - locate or add a pagecache page 687 * @mapping: the page's address_space 688 * @index: the page's index into the mapping 689 * @gfp_mask: page allocation mode 690 * 691 * Locates a page in the pagecache. If the page is not present, a new page 692 * is allocated using @gfp_mask and is added to the pagecache and to the VM's 693 * LRU list. The returned page is locked and has its reference count 694 * incremented. 695 * 696 * find_or_create_page() may sleep, even if @gfp_flags specifies an atomic 697 * allocation! 698 * 699 * find_or_create_page() returns the desired page's address, or zero on 700 * memory exhaustion. 701 */ 702struct page *find_or_create_page(struct address_space *mapping, 703 pgoff_t index, gfp_t gfp_mask) 704{ 705 struct page *page; 706 int err; 707repeat: 708 page = find_lock_page(mapping, index); 709 if (!page) { 710 page = __page_cache_alloc(gfp_mask); 711 if (!page) 712 return NULL; 713 /* 714 * We want a regular kernel memory (not highmem or DMA etc) 715 * allocation for the radix tree nodes, but we need to honour 716 * the context-specific requirements the caller has asked for. 717 * GFP_RECLAIM_MASK collects those requirements. 718 */ 719 err = add_to_page_cache_lru(page, mapping, index, 720 (gfp_mask & GFP_RECLAIM_MASK)); 721 if (unlikely(err)) { 722 page_cache_release(page); 723 page = NULL; 724 if (err == -EEXIST) 725 goto repeat; 726 } 727 } 728 return page; 729} 730EXPORT_SYMBOL(find_or_create_page); 731 732/** 733 * find_get_pages - gang pagecache lookup 734 * @mapping: The address_space to search 735 * @start: The starting page index 736 * @nr_pages: The maximum number of pages 737 * @pages: Where the resulting pages are placed 738 * 739 * find_get_pages() will search for and return a group of up to 740 * @nr_pages pages in the mapping. The pages are placed at @pages. 741 * find_get_pages() takes a reference against the returned pages. 742 * 743 * The search returns a group of mapping-contiguous pages with ascending 744 * indexes. There may be holes in the indices due to not-present pages. 745 * 746 * find_get_pages() returns the number of pages which were found. 747 */ 748unsigned find_get_pages(struct address_space *mapping, pgoff_t start, 749 unsigned int nr_pages, struct page **pages) 750{ 751 unsigned int i; 752 unsigned int ret; 753 unsigned int nr_found; 754 755 rcu_read_lock(); 756restart: 757 nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree, 758 (void ***)pages, start, nr_pages); 759 ret = 0; 760 for (i = 0; i < nr_found; i++) { 761 struct page *page; 762repeat: 763 page = radix_tree_deref_slot((void **)pages[i]); 764 if (unlikely(!page)) 765 continue; 766 /* 767 * this can only trigger if nr_found == 1, making livelock 768 * a non issue. 769 */ 770 if (unlikely(page == RADIX_TREE_RETRY)) 771 goto restart; 772 773 if (!page_cache_get_speculative(page)) 774 goto repeat; 775 776 /* Has the page moved? */ 777 if (unlikely(page != *((void **)pages[i]))) { 778 page_cache_release(page); 779 goto repeat; 780 } 781 782 pages[ret] = page; 783 ret++; 784 } 785 rcu_read_unlock(); 786 return ret; 787} 788 789/** 790 * find_get_pages_contig - gang contiguous pagecache lookup 791 * @mapping: The address_space to search 792 * @index: The starting page index 793 * @nr_pages: The maximum number of pages 794 * @pages: Where the resulting pages are placed 795 * 796 * find_get_pages_contig() works exactly like find_get_pages(), except 797 * that the returned number of pages are guaranteed to be contiguous. 798 * 799 * find_get_pages_contig() returns the number of pages which were found. 800 */ 801unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, 802 unsigned int nr_pages, struct page **pages) 803{ 804 unsigned int i; 805 unsigned int ret; 806 unsigned int nr_found; 807 808 rcu_read_lock(); 809restart: 810 nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree, 811 (void ***)pages, index, nr_pages); 812 ret = 0; 813 for (i = 0; i < nr_found; i++) { 814 struct page *page; 815repeat: 816 page = radix_tree_deref_slot((void **)pages[i]); 817 if (unlikely(!page)) 818 continue; 819 /* 820 * this can only trigger if nr_found == 1, making livelock 821 * a non issue. 822 */ 823 if (unlikely(page == RADIX_TREE_RETRY)) 824 goto restart; 825 826 if (page->mapping == NULL || page->index != index) 827 break; 828 829 if (!page_cache_get_speculative(page)) 830 goto repeat; 831 832 /* Has the page moved? */ 833 if (unlikely(page != *((void **)pages[i]))) { 834 page_cache_release(page); 835 goto repeat; 836 } 837 838 pages[ret] = page; 839 ret++; 840 index++; 841 } 842 rcu_read_unlock(); 843 return ret; 844} 845EXPORT_SYMBOL(find_get_pages_contig); 846 847/** 848 * find_get_pages_tag - find and return pages that match @tag 849 * @mapping: the address_space to search 850 * @index: the starting page index 851 * @tag: the tag index 852 * @nr_pages: the maximum number of pages 853 * @pages: where the resulting pages are placed 854 * 855 * Like find_get_pages, except we only return pages which are tagged with 856 * @tag. We update @index to index the next page for the traversal. 857 */ 858unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index, 859 int tag, unsigned int nr_pages, struct page **pages) 860{ 861 unsigned int i; 862 unsigned int ret; 863 unsigned int nr_found; 864 865 rcu_read_lock(); 866restart: 867 nr_found = radix_tree_gang_lookup_tag_slot(&mapping->page_tree, 868 (void ***)pages, *index, nr_pages, tag); 869 ret = 0; 870 for (i = 0; i < nr_found; i++) { 871 struct page *page; 872repeat: 873 page = radix_tree_deref_slot((void **)pages[i]); 874 if (unlikely(!page)) 875 continue; 876 /* 877 * this can only trigger if nr_found == 1, making livelock 878 * a non issue. 879 */ 880 if (unlikely(page == RADIX_TREE_RETRY)) 881 goto restart; 882 883 if (!page_cache_get_speculative(page)) 884 goto repeat; 885 886 /* Has the page moved? */ 887 if (unlikely(page != *((void **)pages[i]))) { 888 page_cache_release(page); 889 goto repeat; 890 } 891 892 pages[ret] = page; 893 ret++; 894 } 895 rcu_read_unlock(); 896 897 if (ret) 898 *index = pages[ret - 1]->index + 1; 899 900 return ret; 901} 902EXPORT_SYMBOL(find_get_pages_tag); 903 904/** 905 * grab_cache_page_nowait - returns locked page at given index in given cache 906 * @mapping: target address_space 907 * @index: the page index 908 * 909 * Same as grab_cache_page(), but do not wait if the page is unavailable. 910 * This is intended for speculative data generators, where the data can 911 * be regenerated if the page couldn't be grabbed. This routine should 912 * be safe to call while holding the lock for another page. 913 * 914 * Clear __GFP_FS when allocating the page to avoid recursion into the fs 915 * and deadlock against the caller's locked page. 916 */ 917struct page * 918grab_cache_page_nowait(struct address_space *mapping, pgoff_t index) 919{ 920 struct page *page = find_get_page(mapping, index); 921 922 if (page) { 923 if (trylock_page(page)) 924 return page; 925 page_cache_release(page); 926 return NULL; 927 } 928 page = __page_cache_alloc(mapping_gfp_mask(mapping) & ~__GFP_FS); 929 if (page && add_to_page_cache_lru(page, mapping, index, GFP_NOFS)) { 930 page_cache_release(page); 931 page = NULL; 932 } 933 return page; 934} 935EXPORT_SYMBOL(grab_cache_page_nowait); 936 937/* 938 * CD/DVDs are error prone. When a medium error occurs, the driver may fail 939 * a _large_ part of the i/o request. Imagine the worst scenario: 940 * 941 * ---R__________________________________________B__________ 942 * ^ reading here ^ bad block(assume 4k) 943 * 944 * read(R) => miss => readahead(R...B) => media error => frustrating retries 945 * => failing the whole request => read(R) => read(R+1) => 946 * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) => 947 * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) => 948 * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ...... 949 * 950 * It is going insane. Fix it by quickly scaling down the readahead size. 951 */ 952static void shrink_readahead_size_eio(struct file *filp, 953 struct file_ra_state *ra) 954{ 955 ra->ra_pages /= 4; 956} 957 958/** 959 * do_generic_file_read - generic file read routine 960 * @filp: the file to read 961 * @ppos: current file position 962 * @desc: read_descriptor 963 * @actor: read method 964 * 965 * This is a generic file read routine, and uses the 966 * mapping->a_ops->readpage() function for the actual low-level stuff. 967 * 968 * This is really ugly. But the goto's actually try to clarify some 969 * of the logic when it comes to error handling etc. 970 */ 971static void do_generic_file_read(struct file *filp, loff_t *ppos, 972 read_descriptor_t *desc, read_actor_t actor) 973{ 974 struct address_space *mapping = filp->f_mapping; 975 struct inode *inode = mapping->host; 976 struct file_ra_state *ra = &filp->f_ra; 977 pgoff_t index; 978 pgoff_t last_index; 979 pgoff_t prev_index; 980 unsigned long offset; /* offset into pagecache page */ 981 unsigned int prev_offset; 982 int error; 983 984 index = *ppos >> PAGE_CACHE_SHIFT; 985 prev_index = ra->prev_pos >> PAGE_CACHE_SHIFT; 986 prev_offset = ra->prev_pos & (PAGE_CACHE_SIZE-1); 987 last_index = (*ppos + desc->count + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT; 988 offset = *ppos & ~PAGE_CACHE_MASK; 989 990 for (;;) { 991 struct page *page; 992 pgoff_t end_index; 993 loff_t isize; 994 unsigned long nr, ret; 995 996 cond_resched(); 997find_page: 998 page = find_get_page(mapping, index); 999 if (!page) { 1000 page_cache_sync_readahead(mapping, 1001 ra, filp, 1002 index, last_index - index); 1003 page = find_get_page(mapping, index); 1004 if (unlikely(page == NULL)) 1005 goto no_cached_page; 1006 } 1007 if (PageReadahead(page)) { 1008 page_cache_async_readahead(mapping, 1009 ra, filp, page, 1010 index, last_index - index); 1011 } 1012 if (!PageUptodate(page)) { 1013 if (inode->i_blkbits == PAGE_CACHE_SHIFT || 1014 !mapping->a_ops->is_partially_uptodate) 1015 goto page_not_up_to_date; 1016 if (!trylock_page(page)) 1017 goto page_not_up_to_date; 1018 if (!mapping->a_ops->is_partially_uptodate(page, 1019 desc, offset)) 1020 goto page_not_up_to_date_locked; 1021 unlock_page(page); 1022 } 1023page_ok: 1024 /* 1025 * i_size must be checked after we know the page is Uptodate. 1026 * 1027 * Checking i_size after the check allows us to calculate 1028 * the correct value for "nr", which means the zero-filled 1029 * part of the page is not copied back to userspace (unless 1030 * another truncate extends the file - this is desired though). 1031 */ 1032 1033 isize = i_size_read(inode); 1034 end_index = (isize - 1) >> PAGE_CACHE_SHIFT; 1035 if (unlikely(!isize || index > end_index)) { 1036 page_cache_release(page); 1037 goto out; 1038 } 1039 1040 /* nr is the maximum number of bytes to copy from this page */ 1041 nr = PAGE_CACHE_SIZE; 1042 if (index == end_index) { 1043 nr = ((isize - 1) & ~PAGE_CACHE_MASK) + 1; 1044 if (nr <= offset) { 1045 page_cache_release(page); 1046 goto out; 1047 } 1048 } 1049 nr = nr - offset; 1050 1051 /* If users can be writing to this page using arbitrary 1052 * virtual addresses, take care about potential aliasing 1053 * before reading the page on the kernel side. 1054 */ 1055 if (mapping_writably_mapped(mapping)) 1056 flush_dcache_page(page); 1057 1058 /* 1059 * When a sequential read accesses a page several times, 1060 * only mark it as accessed the first time. 1061 */ 1062 if (prev_index != index || offset != prev_offset) 1063 mark_page_accessed(page); 1064 prev_index = index; 1065 1066 /* 1067 * Ok, we have the page, and it's up-to-date, so 1068 * now we can copy it to user space... 1069 * 1070 * The actor routine returns how many bytes were actually used.. 1071 * NOTE! This may not be the same as how much of a user buffer 1072 * we filled up (we may be padding etc), so we can only update 1073 * "pos" here (the actor routine has to update the user buffer 1074 * pointers and the remaining count). 1075 */ 1076 ret = actor(desc, page, offset, nr); 1077 offset += ret; 1078 index += offset >> PAGE_CACHE_SHIFT; 1079 offset &= ~PAGE_CACHE_MASK; 1080 prev_offset = offset; 1081 1082 page_cache_release(page); 1083 if (ret == nr && desc->count) 1084 continue; 1085 goto out; 1086 1087page_not_up_to_date: 1088 /* Get exclusive access to the page ... */ 1089 error = lock_page_killable(page); 1090 if (unlikely(error)) 1091 goto readpage_error; 1092 1093page_not_up_to_date_locked: 1094 /* Did it get truncated before we got the lock? */ 1095 if (!page->mapping) { 1096 unlock_page(page); 1097 page_cache_release(page); 1098 continue; 1099 } 1100 1101 /* Did somebody else fill it already? */ 1102 if (PageUptodate(page)) { 1103 unlock_page(page); 1104 goto page_ok; 1105 } 1106 1107readpage: 1108 /* Start the actual read. The read will unlock the page. */ 1109 error = mapping->a_ops->readpage(filp, page); 1110 1111 if (unlikely(error)) { 1112 if (error == AOP_TRUNCATED_PAGE) { 1113 page_cache_release(page); 1114 goto find_page; 1115 } 1116 goto readpage_error; 1117 } 1118 1119 if (!PageUptodate(page)) { 1120 error = lock_page_killable(page); 1121 if (unlikely(error)) 1122 goto readpage_error; 1123 if (!PageUptodate(page)) { 1124 if (page->mapping == NULL) { 1125 /* 1126 * invalidate_mapping_pages got it 1127 */ 1128 unlock_page(page); 1129 page_cache_release(page); 1130 goto find_page; 1131 } 1132 unlock_page(page); 1133 shrink_readahead_size_eio(filp, ra); 1134 error = -EIO; 1135 goto readpage_error; 1136 } 1137 unlock_page(page); 1138 } 1139 1140 goto page_ok; 1141 1142readpage_error: 1143 /* UHHUH! A synchronous read error occurred. Report it */ 1144 desc->error = error; 1145 page_cache_release(page); 1146 goto out; 1147 1148no_cached_page: 1149 /* 1150 * Ok, it wasn't cached, so we need to create a new 1151 * page.. 1152 */ 1153 page = page_cache_alloc_cold(mapping); 1154 if (!page) { 1155 desc->error = -ENOMEM; 1156 goto out; 1157 } 1158 error = add_to_page_cache_lru(page, mapping, 1159 index, GFP_KERNEL); 1160 if (error) { 1161 page_cache_release(page); 1162 if (error == -EEXIST) 1163 goto find_page; 1164 desc->error = error; 1165 goto out; 1166 } 1167 goto readpage; 1168 } 1169 1170out: 1171 ra->prev_pos = prev_index; 1172 ra->prev_pos <<= PAGE_CACHE_SHIFT; 1173 ra->prev_pos |= prev_offset; 1174 1175 *ppos = ((loff_t)index << PAGE_CACHE_SHIFT) + offset; 1176 file_accessed(filp); 1177} 1178 1179int file_read_actor(read_descriptor_t *desc, struct page *page, 1180 unsigned long offset, unsigned long size) 1181{ 1182 char *kaddr; 1183 unsigned long left, count = desc->count; 1184 1185 if (size > count) 1186 size = count; 1187 1188 /* 1189 * Faults on the destination of a read are common, so do it before 1190 * taking the kmap. 1191 */ 1192 if (!fault_in_pages_writeable(desc->arg.buf, size)) { 1193 kaddr = kmap_atomic(page, KM_USER0); 1194 left = __copy_to_user_inatomic(desc->arg.buf, 1195 kaddr + offset, size); 1196 kunmap_atomic(kaddr, KM_USER0); 1197 if (left == 0) 1198 goto success; 1199 } 1200 1201 /* Do it the slow way */ 1202 kaddr = kmap(page); 1203 left = __copy_to_user(desc->arg.buf, kaddr + offset, size); 1204 kunmap(page); 1205 1206 if (left) { 1207 size -= left; 1208 desc->error = -EFAULT; 1209 } 1210success: 1211 desc->count = count - size; 1212 desc->written += size; 1213 desc->arg.buf += size; 1214 return size; 1215} 1216 1217/* 1218 * Performs necessary checks before doing a write 1219 * @iov: io vector request 1220 * @nr_segs: number of segments in the iovec 1221 * @count: number of bytes to write 1222 * @access_flags: type of access: %VERIFY_READ or %VERIFY_WRITE 1223 * 1224 * Adjust number of segments and amount of bytes to write (nr_segs should be 1225 * properly initialized first). Returns appropriate error code that caller 1226 * should return or zero in case that write should be allowed. 1227 */ 1228int generic_segment_checks(const struct iovec *iov, 1229 unsigned long *nr_segs, size_t *count, int access_flags) 1230{ 1231 unsigned long seg; 1232 size_t cnt = 0; 1233 for (seg = 0; seg < *nr_segs; seg++) { 1234 const struct iovec *iv = &iov[seg]; 1235 1236 /* 1237 * If any segment has a negative length, or the cumulative 1238 * length ever wraps negative then return -EINVAL. 1239 */ 1240 cnt += iv->iov_len; 1241 if (unlikely((ssize_t)(cnt|iv->iov_len) < 0)) 1242 return -EINVAL; 1243 if (access_ok(access_flags, iv->iov_base, iv->iov_len)) 1244 continue; 1245 if (seg == 0) 1246 return -EFAULT; 1247 *nr_segs = seg; 1248 cnt -= iv->iov_len; /* This segment is no good */ 1249 break; 1250 } 1251 *count = cnt; 1252 return 0; 1253} 1254EXPORT_SYMBOL(generic_segment_checks); 1255 1256/** 1257 * generic_file_aio_read - generic filesystem read routine 1258 * @iocb: kernel I/O control block 1259 * @iov: io vector request 1260 * @nr_segs: number of segments in the iovec 1261 * @pos: current file position 1262 * 1263 * This is the "read()" routine for all filesystems 1264 * that can use the page cache directly. 1265 */ 1266ssize_t 1267generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov, 1268 unsigned long nr_segs, loff_t pos) 1269{ 1270 struct file *filp = iocb->ki_filp; 1271 ssize_t retval; 1272 unsigned long seg; 1273 size_t count; 1274 loff_t *ppos = &iocb->ki_pos; 1275 1276 count = 0; 1277 retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE); 1278 if (retval) 1279 return retval; 1280 1281 /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */ 1282 if (filp->f_flags & O_DIRECT) { 1283 loff_t size; 1284 struct address_space *mapping; 1285 struct inode *inode; 1286 1287 mapping = filp->f_mapping; 1288 inode = mapping->host; 1289 if (!count) 1290 goto out; /* skip atime */ 1291 size = i_size_read(inode); 1292 if (pos < size) { 1293 retval = filemap_write_and_wait_range(mapping, pos, 1294 pos + iov_length(iov, nr_segs) - 1); 1295 if (!retval) { 1296 retval = mapping->a_ops->direct_IO(READ, iocb, 1297 iov, pos, nr_segs); 1298 } 1299 if (retval > 0) 1300 *ppos = pos + retval; 1301 if (retval) { 1302 file_accessed(filp); 1303 goto out; 1304 } 1305 } 1306 } 1307 1308 for (seg = 0; seg < nr_segs; seg++) { 1309 read_descriptor_t desc; 1310 1311 desc.written = 0; 1312 desc.arg.buf = iov[seg].iov_base; 1313 desc.count = iov[seg].iov_len; 1314 if (desc.count == 0) 1315 continue; 1316 desc.error = 0; 1317 do_generic_file_read(filp, ppos, &desc, file_read_actor); 1318 retval += desc.written; 1319 if (desc.error) { 1320 retval = retval ?: desc.error; 1321 break; 1322 } 1323 if (desc.count > 0) 1324 break; 1325 } 1326out: 1327 return retval; 1328} 1329EXPORT_SYMBOL(generic_file_aio_read); 1330 1331static ssize_t 1332do_readahead(struct address_space *mapping, struct file *filp, 1333 pgoff_t index, unsigned long nr) 1334{ 1335 if (!mapping || !mapping->a_ops || !mapping->a_ops->readpage) 1336 return -EINVAL; 1337 1338 force_page_cache_readahead(mapping, filp, index, nr); 1339 return 0; 1340} 1341 1342SYSCALL_DEFINE(readahead)(int fd, loff_t offset, size_t count) 1343{ 1344 ssize_t ret; 1345 struct file *file; 1346 1347 ret = -EBADF; 1348 file = fget(fd); 1349 if (file) { 1350 if (file->f_mode & FMODE_READ) { 1351 struct address_space *mapping = file->f_mapping; 1352 pgoff_t start = offset >> PAGE_CACHE_SHIFT; 1353 pgoff_t end = (offset + count - 1) >> PAGE_CACHE_SHIFT; 1354 unsigned long len = end - start + 1; 1355 ret = do_readahead(mapping, file, start, len); 1356 } 1357 fput(file); 1358 } 1359 return ret; 1360} 1361#ifdef CONFIG_HAVE_SYSCALL_WRAPPERS 1362asmlinkage long SyS_readahead(long fd, loff_t offset, long count) 1363{ 1364 return SYSC_readahead((int) fd, offset, (size_t) count); 1365} 1366SYSCALL_ALIAS(sys_readahead, SyS_readahead); 1367#endif 1368 1369#ifdef CONFIG_MMU 1370/** 1371 * page_cache_read - adds requested page to the page cache if not already there 1372 * @file: file to read 1373 * @offset: page index 1374 * 1375 * This adds the requested page to the page cache if it isn't already there, 1376 * and schedules an I/O to read in its contents from disk. 1377 */ 1378static int page_cache_read(struct file *file, pgoff_t offset) 1379{ 1380 struct address_space *mapping = file->f_mapping; 1381 struct page *page; 1382 int ret; 1383 1384 do { 1385 page = page_cache_alloc_cold(mapping); 1386 if (!page) 1387 return -ENOMEM; 1388 1389 ret = add_to_page_cache_lru(page, mapping, offset, GFP_KERNEL); 1390 if (ret == 0) 1391 ret = mapping->a_ops->readpage(file, page); 1392 else if (ret == -EEXIST) 1393 ret = 0; /* losing race to add is OK */ 1394 1395 page_cache_release(page); 1396 1397 } while (ret == AOP_TRUNCATED_PAGE); 1398 1399 return ret; 1400} 1401 1402#define MMAP_LOTSAMISS (100) 1403 1404/* 1405 * Synchronous readahead happens when we don't even find 1406 * a page in the page cache at all. 1407 */ 1408static void do_sync_mmap_readahead(struct vm_area_struct *vma, 1409 struct file_ra_state *ra, 1410 struct file *file, 1411 pgoff_t offset) 1412{ 1413 unsigned long ra_pages; 1414 struct address_space *mapping = file->f_mapping; 1415 1416 /* If we don't want any read-ahead, don't bother */ 1417 if (VM_RandomReadHint(vma)) 1418 return; 1419 1420 if (VM_SequentialReadHint(vma) || 1421 offset - 1 == (ra->prev_pos >> PAGE_CACHE_SHIFT)) { 1422 page_cache_sync_readahead(mapping, ra, file, offset, 1423 ra->ra_pages); 1424 return; 1425 } 1426 1427 if (ra->mmap_miss < INT_MAX) 1428 ra->mmap_miss++; 1429 1430 /* 1431 * Do we miss much more than hit in this file? If so, 1432 * stop bothering with read-ahead. It will only hurt. 1433 */ 1434 if (ra->mmap_miss > MMAP_LOTSAMISS) 1435 return; 1436 1437 /* 1438 * mmap read-around 1439 */ 1440 ra_pages = max_sane_readahead(ra->ra_pages); 1441 if (ra_pages) { 1442 ra->start = max_t(long, 0, offset - ra_pages/2); 1443 ra->size = ra_pages; 1444 ra->async_size = 0; 1445 ra_submit(ra, mapping, file); 1446 } 1447} 1448 1449/* 1450 * Asynchronous readahead happens when we find the page and PG_readahead, 1451 * so we want to possibly extend the readahead further.. 1452 */ 1453static void do_async_mmap_readahead(struct vm_area_struct *vma, 1454 struct file_ra_state *ra, 1455 struct file *file, 1456 struct page *page, 1457 pgoff_t offset) 1458{ 1459 struct address_space *mapping = file->f_mapping; 1460 1461 /* If we don't want any read-ahead, don't bother */ 1462 if (VM_RandomReadHint(vma)) 1463 return; 1464 if (ra->mmap_miss > 0) 1465 ra->mmap_miss--; 1466 if (PageReadahead(page)) 1467 page_cache_async_readahead(mapping, ra, file, 1468 page, offset, ra->ra_pages); 1469} 1470 1471/** 1472 * filemap_fault - read in file data for page fault handling 1473 * @vma: vma in which the fault was taken 1474 * @vmf: struct vm_fault containing details of the fault 1475 * 1476 * filemap_fault() is invoked via the vma operations vector for a 1477 * mapped memory region to read in file data during a page fault. 1478 * 1479 * The goto's are kind of ugly, but this streamlines the normal case of having 1480 * it in the page cache, and handles the special cases reasonably without 1481 * having a lot of duplicated code. 1482 */ 1483int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1484{ 1485 int error; 1486 struct file *file = vma->vm_file; 1487 struct address_space *mapping = file->f_mapping; 1488 struct file_ra_state *ra = &file->f_ra; 1489 struct inode *inode = mapping->host; 1490 pgoff_t offset = vmf->pgoff; 1491 struct page *page; 1492 pgoff_t size; 1493 int ret = 0; 1494 1495 size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 1496 if (offset >= size) 1497 return VM_FAULT_SIGBUS; 1498 1499 /* 1500 * Do we have something in the page cache already? 1501 */ 1502 page = find_get_page(mapping, offset); 1503 if (likely(page)) { 1504 /* 1505 * We found the page, so try async readahead before 1506 * waiting for the lock. 1507 */ 1508 do_async_mmap_readahead(vma, ra, file, page, offset); 1509 lock_page(page); 1510 1511 /* Did it get truncated? */ 1512 if (unlikely(page->mapping != mapping)) { 1513 unlock_page(page); 1514 put_page(page); 1515 goto no_cached_page; 1516 } 1517 } else { 1518 /* No page in the page cache at all */ 1519 do_sync_mmap_readahead(vma, ra, file, offset); 1520 count_vm_event(PGMAJFAULT); 1521 ret = VM_FAULT_MAJOR; 1522retry_find: 1523 page = find_lock_page(mapping, offset); 1524 if (!page) 1525 goto no_cached_page; 1526 } 1527 1528 /* 1529 * We have a locked page in the page cache, now we need to check 1530 * that it's up-to-date. If not, it is going to be due to an error. 1531 */ 1532 if (unlikely(!PageUptodate(page))) 1533 goto page_not_uptodate; 1534 1535 /* 1536 * Found the page and have a reference on it. 1537 * We must recheck i_size under page lock. 1538 */ 1539 size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 1540 if (unlikely(offset >= size)) { 1541 unlock_page(page); 1542 page_cache_release(page); 1543 return VM_FAULT_SIGBUS; 1544 } 1545 1546 ra->prev_pos = (loff_t)offset << PAGE_CACHE_SHIFT; 1547 vmf->page = page; 1548 return ret | VM_FAULT_LOCKED; 1549 1550no_cached_page: 1551 /* 1552 * We're only likely to ever get here if MADV_RANDOM is in 1553 * effect. 1554 */ 1555 error = page_cache_read(file, offset); 1556 1557 /* 1558 * The page we want has now been added to the page cache. 1559 * In the unlikely event that someone removed it in the 1560 * meantime, we'll just come back here and read it again. 1561 */ 1562 if (error >= 0) 1563 goto retry_find; 1564 1565 /* 1566 * An error return from page_cache_read can result if the 1567 * system is low on memory, or a problem occurs while trying 1568 * to schedule I/O. 1569 */ 1570 if (error == -ENOMEM) 1571 return VM_FAULT_OOM; 1572 return VM_FAULT_SIGBUS; 1573 1574page_not_uptodate: 1575 /* 1576 * Umm, take care of errors if the page isn't up-to-date. 1577 * Try to re-read it _once_. We do this synchronously, 1578 * because there really aren't any performance issues here 1579 * and we need to check for errors. 1580 */ 1581 ClearPageError(page); 1582 error = mapping->a_ops->readpage(file, page); 1583 if (!error) { 1584 wait_on_page_locked(page); 1585 if (!PageUptodate(page)) 1586 error = -EIO; 1587 } 1588 page_cache_release(page); 1589 1590 if (!error || error == AOP_TRUNCATED_PAGE) 1591 goto retry_find; 1592 1593 /* Things didn't work out. Return zero to tell the mm layer so. */ 1594 shrink_readahead_size_eio(file, ra); 1595 return VM_FAULT_SIGBUS; 1596} 1597EXPORT_SYMBOL(filemap_fault); 1598 1599const struct vm_operations_struct generic_file_vm_ops = { 1600 .fault = filemap_fault, 1601}; 1602 1603/* This is used for a general mmap of a disk file */ 1604 1605int generic_file_mmap(struct file * file, struct vm_area_struct * vma) 1606{ 1607 struct address_space *mapping = file->f_mapping; 1608 1609 if (!mapping->a_ops->readpage) 1610 return -ENOEXEC; 1611 file_accessed(file); 1612 vma->vm_ops = &generic_file_vm_ops; 1613 vma->vm_flags |= VM_CAN_NONLINEAR; 1614 return 0; 1615} 1616 1617/* 1618 * This is for filesystems which do not implement ->writepage. 1619 */ 1620int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma) 1621{ 1622 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) 1623 return -EINVAL; 1624 return generic_file_mmap(file, vma); 1625} 1626#else 1627int generic_file_mmap(struct file * file, struct vm_area_struct * vma) 1628{ 1629 return -ENOSYS; 1630} 1631int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma) 1632{ 1633 return -ENOSYS; 1634} 1635#endif /* CONFIG_MMU */ 1636 1637EXPORT_SYMBOL(generic_file_mmap); 1638EXPORT_SYMBOL(generic_file_readonly_mmap); 1639 1640static struct page *__read_cache_page(struct address_space *mapping, 1641 pgoff_t index, 1642 int (*filler)(void *,struct page*), 1643 void *data, 1644 gfp_t gfp) 1645{ 1646 struct page *page; 1647 int err; 1648repeat: 1649 page = find_get_page(mapping, index); 1650 if (!page) { 1651 page = __page_cache_alloc(gfp | __GFP_COLD); 1652 if (!page) 1653 return ERR_PTR(-ENOMEM); 1654 err = add_to_page_cache_lru(page, mapping, index, GFP_KERNEL); 1655 if (unlikely(err)) { 1656 page_cache_release(page); 1657 if (err == -EEXIST) 1658 goto repeat; 1659 /* Presumably ENOMEM for radix tree node */ 1660 return ERR_PTR(err); 1661 } 1662 err = filler(data, page); 1663 if (err < 0) { 1664 page_cache_release(page); 1665 page = ERR_PTR(err); 1666 } 1667 } 1668 return page; 1669} 1670 1671static struct page *do_read_cache_page(struct address_space *mapping, 1672 pgoff_t index, 1673 int (*filler)(void *,struct page*), 1674 void *data, 1675 gfp_t gfp) 1676 1677{ 1678 struct page *page; 1679 int err; 1680 1681retry: 1682 page = __read_cache_page(mapping, index, filler, data, gfp); 1683 if (IS_ERR(page)) 1684 return page; 1685 if (PageUptodate(page)) 1686 goto out; 1687 1688 lock_page(page); 1689 if (!page->mapping) { 1690 unlock_page(page); 1691 page_cache_release(page); 1692 goto retry; 1693 } 1694 if (PageUptodate(page)) { 1695 unlock_page(page); 1696 goto out; 1697 } 1698 err = filler(data, page); 1699 if (err < 0) { 1700 page_cache_release(page); 1701 return ERR_PTR(err); 1702 } 1703out: 1704 mark_page_accessed(page); 1705 return page; 1706} 1707 1708/** 1709 * read_cache_page_async - read into page cache, fill it if needed 1710 * @mapping: the page's address_space 1711 * @index: the page index 1712 * @filler: function to perform the read 1713 * @data: destination for read data 1714 * 1715 * Same as read_cache_page, but don't wait for page to become unlocked 1716 * after submitting it to the filler. 1717 * 1718 * Read into the page cache. If a page already exists, and PageUptodate() is 1719 * not set, try to fill the page but don't wait for it to become unlocked. 1720 * 1721 * If the page does not get brought uptodate, return -EIO. 1722 */ 1723struct page *read_cache_page_async(struct address_space *mapping, 1724 pgoff_t index, 1725 int (*filler)(void *,struct page*), 1726 void *data) 1727{ 1728 return do_read_cache_page(mapping, index, filler, data, mapping_gfp_mask(mapping)); 1729} 1730EXPORT_SYMBOL(read_cache_page_async); 1731 1732static struct page *wait_on_page_read(struct page *page) 1733{ 1734 if (!IS_ERR(page)) { 1735 wait_on_page_locked(page); 1736 if (!PageUptodate(page)) { 1737 page_cache_release(page); 1738 page = ERR_PTR(-EIO); 1739 } 1740 } 1741 return page; 1742} 1743 1744/** 1745 * read_cache_page_gfp - read into page cache, using specified page allocation flags. 1746 * @mapping: the page's address_space 1747 * @index: the page index 1748 * @gfp: the page allocator flags to use if allocating 1749 * 1750 * This is the same as "read_mapping_page(mapping, index, NULL)", but with 1751 * any new page allocations done using the specified allocation flags. Note 1752 * that the Radix tree operations will still use GFP_KERNEL, so you can't 1753 * expect to do this atomically or anything like that - but you can pass in 1754 * other page requirements. 1755 * 1756 * If the page does not get brought uptodate, return -EIO. 1757 */ 1758struct page *read_cache_page_gfp(struct address_space *mapping, 1759 pgoff_t index, 1760 gfp_t gfp) 1761{ 1762 filler_t *filler = (filler_t *)mapping->a_ops->readpage; 1763 1764 return wait_on_page_read(do_read_cache_page(mapping, index, filler, NULL, gfp)); 1765} 1766EXPORT_SYMBOL(read_cache_page_gfp); 1767 1768/** 1769 * read_cache_page - read into page cache, fill it if needed 1770 * @mapping: the page's address_space 1771 * @index: the page index 1772 * @filler: function to perform the read 1773 * @data: destination for read data 1774 * 1775 * Read into the page cache. If a page already exists, and PageUptodate() is 1776 * not set, try to fill the page then wait for it to become unlocked. 1777 * 1778 * If the page does not get brought uptodate, return -EIO. 1779 */ 1780struct page *read_cache_page(struct address_space *mapping, 1781 pgoff_t index, 1782 int (*filler)(void *,struct page*), 1783 void *data) 1784{ 1785 return wait_on_page_read(read_cache_page_async(mapping, index, filler, data)); 1786} 1787EXPORT_SYMBOL(read_cache_page); 1788 1789/* 1790 * The logic we want is 1791 * 1792 * if suid or (sgid and xgrp) 1793 * remove privs 1794 */ 1795int should_remove_suid(struct dentry *dentry) 1796{ 1797 mode_t mode = dentry->d_inode->i_mode; 1798 int kill = 0; 1799 1800 /* suid always must be killed */ 1801 if (unlikely(mode & S_ISUID)) 1802 kill = ATTR_KILL_SUID; 1803 1804 /* 1805 * sgid without any exec bits is just a mandatory locking mark; leave 1806 * it alone. If some exec bits are set, it's a real sgid; kill it. 1807 */ 1808 if (unlikely((mode & S_ISGID) && (mode & S_IXGRP))) 1809 kill |= ATTR_KILL_SGID; 1810 1811 if (unlikely(kill && !capable(CAP_FSETID) && S_ISREG(mode))) 1812 return kill; 1813 1814 return 0; 1815} 1816EXPORT_SYMBOL(should_remove_suid); 1817 1818static int __remove_suid(struct dentry *dentry, int kill) 1819{ 1820 struct iattr newattrs; 1821 1822 newattrs.ia_valid = ATTR_FORCE | kill; 1823 return notify_change(dentry, &newattrs); 1824} 1825 1826int file_remove_suid(struct file *file) 1827{ 1828 struct dentry *dentry = file->f_path.dentry; 1829 int killsuid = should_remove_suid(dentry); 1830 int killpriv = security_inode_need_killpriv(dentry); 1831 int error = 0; 1832 1833 if (killpriv < 0) 1834 return killpriv; 1835 if (killpriv) 1836 error = security_inode_killpriv(dentry); 1837 if (!error && killsuid) 1838 error = __remove_suid(dentry, killsuid); 1839 1840 return error; 1841} 1842EXPORT_SYMBOL(file_remove_suid); 1843 1844static size_t __iovec_copy_from_user_inatomic(char *vaddr, 1845 const struct iovec *iov, size_t base, size_t bytes) 1846{ 1847 size_t copied = 0, left = 0; 1848 1849 while (bytes) { 1850 char __user *buf = iov->iov_base + base; 1851 int copy = min(bytes, iov->iov_len - base); 1852 1853 base = 0; 1854 left = __copy_from_user_inatomic(vaddr, buf, copy); 1855 copied += copy; 1856 bytes -= copy; 1857 vaddr += copy; 1858 iov++; 1859 1860 if (unlikely(left)) 1861 break; 1862 } 1863 return copied - left; 1864} 1865 1866/* 1867 * Copy as much as we can into the page and return the number of bytes which 1868 * were successfully copied. If a fault is encountered then return the number of 1869 * bytes which were copied. 1870 */ 1871size_t iov_iter_copy_from_user_atomic(struct page *page, 1872 struct iov_iter *i, unsigned long offset, size_t bytes) 1873{ 1874 char *kaddr; 1875 size_t copied; 1876 1877 BUG_ON(!in_atomic()); 1878 kaddr = kmap_atomic(page, KM_USER0); 1879 if (likely(i->nr_segs == 1)) { 1880 int left; 1881 char __user *buf = i->iov->iov_base + i->iov_offset; 1882 left = __copy_from_user_inatomic(kaddr + offset, buf, bytes); 1883 copied = bytes - left; 1884 } else { 1885 copied = __iovec_copy_from_user_inatomic(kaddr + offset, 1886 i->iov, i->iov_offset, bytes); 1887 } 1888 kunmap_atomic(kaddr, KM_USER0); 1889 1890 return copied; 1891} 1892EXPORT_SYMBOL(iov_iter_copy_from_user_atomic); 1893 1894/* 1895 * This has the same sideeffects and return value as 1896 * iov_iter_copy_from_user_atomic(). 1897 * The difference is that it attempts to resolve faults. 1898 * Page must not be locked. 1899 */ 1900size_t iov_iter_copy_from_user(struct page *page, 1901 struct iov_iter *i, unsigned long offset, size_t bytes) 1902{ 1903 char *kaddr; 1904 size_t copied; 1905 1906 kaddr = kmap(page); 1907 if (likely(i->nr_segs == 1)) { 1908 int left; 1909 char __user *buf = i->iov->iov_base + i->iov_offset; 1910 left = __copy_from_user(kaddr + offset, buf, bytes); 1911 copied = bytes - left; 1912 } else { 1913 copied = __iovec_copy_from_user_inatomic(kaddr + offset, 1914 i->iov, i->iov_offset, bytes); 1915 } 1916 kunmap(page); 1917 return copied; 1918} 1919EXPORT_SYMBOL(iov_iter_copy_from_user); 1920 1921void iov_iter_advance(struct iov_iter *i, size_t bytes) 1922{ 1923 BUG_ON(i->count < bytes); 1924 1925 if (likely(i->nr_segs == 1)) { 1926 i->iov_offset += bytes; 1927 i->count -= bytes; 1928 } else { 1929 const struct iovec *iov = i->iov; 1930 size_t base = i->iov_offset; 1931 1932 /* 1933 * The !iov->iov_len check ensures we skip over unlikely 1934 * zero-length segments (without overruning the iovec). 1935 */ 1936 while (bytes || unlikely(i->count && !iov->iov_len)) { 1937 int copy; 1938 1939 copy = min(bytes, iov->iov_len - base); 1940 BUG_ON(!i->count || i->count < copy); 1941 i->count -= copy; 1942 bytes -= copy; 1943 base += copy; 1944 if (iov->iov_len == base) { 1945 iov++; 1946 base = 0; 1947 } 1948 } 1949 i->iov = iov; 1950 i->iov_offset = base; 1951 } 1952} 1953EXPORT_SYMBOL(iov_iter_advance); 1954 1955/* 1956 * Fault in the first iovec of the given iov_iter, to a maximum length 1957 * of bytes. Returns 0 on success, or non-zero if the memory could not be 1958 * accessed (ie. because it is an invalid address). 1959 * 1960 * writev-intensive code may want this to prefault several iovecs -- that 1961 * would be possible (callers must not rely on the fact that _only_ the 1962 * first iovec will be faulted with the current implementation). 1963 */ 1964int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes) 1965{ 1966 char __user *buf = i->iov->iov_base + i->iov_offset; 1967 bytes = min(bytes, i->iov->iov_len - i->iov_offset); 1968 return fault_in_pages_readable(buf, bytes); 1969} 1970EXPORT_SYMBOL(iov_iter_fault_in_readable); 1971 1972/* 1973 * Return the count of just the current iov_iter segment. 1974 */ 1975size_t iov_iter_single_seg_count(struct iov_iter *i) 1976{ 1977 const struct iovec *iov = i->iov; 1978 if (i->nr_segs == 1) 1979 return i->count; 1980 else 1981 return min(i->count, iov->iov_len - i->iov_offset); 1982} 1983EXPORT_SYMBOL(iov_iter_single_seg_count); 1984 1985/* 1986 * Performs necessary checks before doing a write 1987 * 1988 * Can adjust writing position or amount of bytes to write. 1989 * Returns appropriate error code that caller should return or 1990 * zero in case that write should be allowed. 1991 */ 1992inline int generic_write_checks(struct file *file, loff_t *pos, size_t *count, int isblk) 1993{ 1994 struct inode *inode = file->f_mapping->host; 1995 unsigned long limit = rlimit(RLIMIT_FSIZE); 1996 1997 if (unlikely(*pos < 0)) 1998 return -EINVAL; 1999 2000 if (!isblk) { 2001 /* FIXME: this is for backwards compatibility with 2.4 */ 2002 if (file->f_flags & O_APPEND) 2003 *pos = i_size_read(inode); 2004 2005 if (limit != RLIM_INFINITY) { 2006 if (*pos >= limit) { 2007 send_sig(SIGXFSZ, current, 0); 2008 return -EFBIG; 2009 } 2010 if (*count > limit - (typeof(limit))*pos) { 2011 *count = limit - (typeof(limit))*pos; 2012 } 2013 } 2014 } 2015 2016 /* 2017 * LFS rule 2018 */ 2019 if (unlikely(*pos + *count > MAX_NON_LFS && 2020 !(file->f_flags & O_LARGEFILE))) { 2021 if (*pos >= MAX_NON_LFS) { 2022 return -EFBIG; 2023 } 2024 if (*count > MAX_NON_LFS - (unsigned long)*pos) { 2025 *count = MAX_NON_LFS - (unsigned long)*pos; 2026 } 2027 } 2028 2029 /* 2030 * Are we about to exceed the fs block limit ? 2031 * 2032 * If we have written data it becomes a short write. If we have 2033 * exceeded without writing data we send a signal and return EFBIG. 2034 * Linus frestrict idea will clean these up nicely.. 2035 */ 2036 if (likely(!isblk)) { 2037 if (unlikely(*pos >= inode->i_sb->s_maxbytes)) { 2038 if (*count || *pos > inode->i_sb->s_maxbytes) { 2039 return -EFBIG; 2040 } 2041 /* zero-length writes at ->s_maxbytes are OK */ 2042 } 2043 2044 if (unlikely(*pos + *count > inode->i_sb->s_maxbytes)) 2045 *count = inode->i_sb->s_maxbytes - *pos; 2046 } else { 2047#ifdef CONFIG_BLOCK 2048 loff_t isize; 2049 if (bdev_read_only(I_BDEV(inode))) 2050 return -EPERM; 2051 isize = i_size_read(inode); 2052 if (*pos >= isize) { 2053 if (*count || *pos > isize) 2054 return -ENOSPC; 2055 } 2056 2057 if (*pos + *count > isize) 2058 *count = isize - *pos; 2059#else 2060 return -EPERM; 2061#endif 2062 } 2063 return 0; 2064} 2065EXPORT_SYMBOL(generic_write_checks); 2066 2067int pagecache_write_begin(struct file *file, struct address_space *mapping, 2068 loff_t pos, unsigned len, unsigned flags, 2069 struct page **pagep, void **fsdata) 2070{ 2071 const struct address_space_operations *aops = mapping->a_ops; 2072 2073 return aops->write_begin(file, mapping, pos, len, flags, 2074 pagep, fsdata); 2075} 2076EXPORT_SYMBOL(pagecache_write_begin); 2077 2078int pagecache_write_end(struct file *file, struct address_space *mapping, 2079 loff_t pos, unsigned len, unsigned copied, 2080 struct page *page, void *fsdata) 2081{ 2082 const struct address_space_operations *aops = mapping->a_ops; 2083 2084 mark_page_accessed(page); 2085 return aops->write_end(file, mapping, pos, len, copied, page, fsdata); 2086} 2087EXPORT_SYMBOL(pagecache_write_end); 2088 2089ssize_t 2090generic_file_direct_write(struct kiocb *iocb, const struct iovec *iov, 2091 unsigned long *nr_segs, loff_t pos, loff_t *ppos, 2092 size_t count, size_t ocount) 2093{ 2094 struct file *file = iocb->ki_filp; 2095 struct address_space *mapping = file->f_mapping; 2096 struct inode *inode = mapping->host; 2097 ssize_t written; 2098 size_t write_len; 2099 pgoff_t end; 2100 2101 if (count != ocount) 2102 *nr_segs = iov_shorten((struct iovec *)iov, *nr_segs, count); 2103 2104 write_len = iov_length(iov, *nr_segs); 2105 end = (pos + write_len - 1) >> PAGE_CACHE_SHIFT; 2106 2107 written = filemap_write_and_wait_range(mapping, pos, pos + write_len - 1); 2108 if (written) 2109 goto out; 2110 2111 /* 2112 * After a write we want buffered reads to be sure to go to disk to get 2113 * the new data. We invalidate clean cached page from the region we're 2114 * about to write. We do this *before* the write so that we can return 2115 * without clobbering -EIOCBQUEUED from ->direct_IO(). 2116 */ 2117 if (mapping->nrpages) { 2118 written = invalidate_inode_pages2_range(mapping, 2119 pos >> PAGE_CACHE_SHIFT, end); 2120 /* 2121 * If a page can not be invalidated, return 0 to fall back 2122 * to buffered write. 2123 */ 2124 if (written) { 2125 if (written == -EBUSY) 2126 return 0; 2127 goto out; 2128 } 2129 } 2130 2131 written = mapping->a_ops->direct_IO(WRITE, iocb, iov, pos, *nr_segs); 2132 2133 /* 2134 * Finally, try again to invalidate clean pages which might have been 2135 * cached by non-direct readahead, or faulted in by get_user_pages() 2136 * if the source of the write was an mmap'ed region of the file 2137 * we're writing. Either one is a pretty crazy thing to do, 2138 * so we don't support it 100%. If this invalidation 2139 * fails, tough, the write still worked... 2140 */ 2141 if (mapping->nrpages) { 2142 invalidate_inode_pages2_range(mapping, 2143 pos >> PAGE_CACHE_SHIFT, end); 2144 } 2145 2146 if (written > 0) { 2147 loff_t end = pos + written; 2148 if (end > i_size_read(inode) && !S_ISBLK(inode->i_mode)) { 2149 i_size_write(inode, end); 2150 mark_inode_dirty(inode); 2151 } 2152 *ppos = end; 2153 } 2154out: 2155 return written; 2156} 2157EXPORT_SYMBOL(generic_file_direct_write); 2158 2159/* 2160 * Find or create a page at the given pagecache position. Return the locked 2161 * page. This function is specifically for buffered writes. 2162 */ 2163struct page *grab_cache_page_write_begin(struct address_space *mapping, 2164 pgoff_t index, unsigned flags) 2165{ 2166 int status; 2167 struct page *page; 2168 gfp_t gfp_notmask = 0; 2169 if (flags & AOP_FLAG_NOFS) 2170 gfp_notmask = __GFP_FS; 2171repeat: 2172 page = find_lock_page(mapping, index); 2173 if (likely(page)) 2174 return page; 2175 2176 page = __page_cache_alloc(mapping_gfp_mask(mapping) & ~gfp_notmask); 2177 if (!page) 2178 return NULL; 2179 status = add_to_page_cache_lru(page, mapping, index, 2180 GFP_KERNEL & ~gfp_notmask); 2181 if (unlikely(status)) { 2182 page_cache_release(page); 2183 if (status == -EEXIST) 2184 goto repeat; 2185 return NULL; 2186 } 2187 return page; 2188} 2189EXPORT_SYMBOL(grab_cache_page_write_begin); 2190 2191static ssize_t generic_perform_write(struct file *file, 2192 struct iov_iter *i, loff_t pos) 2193{ 2194 struct address_space *mapping = file->f_mapping; 2195 const struct address_space_operations *a_ops = mapping->a_ops; 2196 long status = 0; 2197 ssize_t written = 0; 2198 unsigned int flags = 0; 2199 2200 /* 2201 * Copies from kernel address space cannot fail (NFSD is a big user). 2202 */ 2203 if (segment_eq(get_fs(), KERNEL_DS)) 2204 flags |= AOP_FLAG_UNINTERRUPTIBLE; 2205 2206 do { 2207 struct page *page; 2208 pgoff_t index; /* Pagecache index for current page */ 2209 unsigned long offset; /* Offset into pagecache page */ 2210 unsigned long bytes; /* Bytes to write to page */ 2211 size_t copied; /* Bytes copied from user */ 2212 void *fsdata; 2213 2214 offset = (pos & (PAGE_CACHE_SIZE - 1)); 2215 index = pos >> PAGE_CACHE_SHIFT; 2216 bytes = min_t(unsigned long, PAGE_CACHE_SIZE - offset, 2217 iov_iter_count(i)); 2218 2219again: 2220 2221 /* 2222 * Bring in the user page that we will copy from _first_. 2223 * Otherwise there's a nasty deadlock on copying from the 2224 * same page as we're writing to, without it being marked 2225 * up-to-date. 2226 * 2227 * Not only is this an optimisation, but it is also required 2228 * to check that the address is actually valid, when atomic 2229 * usercopies are used, below. 2230 */ 2231 if (unlikely(iov_iter_fault_in_readable(i, bytes))) { 2232 status = -EFAULT; 2233 break; 2234 } 2235 2236 status = a_ops->write_begin(file, mapping, pos, bytes, flags, 2237 &page, &fsdata); 2238 if (unlikely(status)) 2239 break; 2240 2241 if (mapping_writably_mapped(mapping)) 2242 flush_dcache_page(page); 2243 2244 pagefault_disable(); 2245 copied = iov_iter_copy_from_user_atomic(page, i, offset, bytes); 2246 pagefault_enable(); 2247 flush_dcache_page(page); 2248 2249 mark_page_accessed(page); 2250 status = a_ops->write_end(file, mapping, pos, bytes, copied, 2251 page, fsdata); 2252 if (unlikely(status < 0)) 2253 break; 2254 copied = status; 2255 2256 cond_resched(); 2257 2258 iov_iter_advance(i, copied); 2259 if (unlikely(copied == 0)) { 2260 /* 2261 * If we were unable to copy any data at all, we must 2262 * fall back to a single segment length write. 2263 * 2264 * If we didn't fallback here, we could livelock 2265 * because not all segments in the iov can be copied at 2266 * once without a pagefault. 2267 */ 2268 bytes = min_t(unsigned long, PAGE_CACHE_SIZE - offset, 2269 iov_iter_single_seg_count(i)); 2270 goto again; 2271 } 2272 pos += copied; 2273 written += copied; 2274 2275 balance_dirty_pages_ratelimited(mapping); 2276 2277 } while (iov_iter_count(i)); 2278 2279 return written ? written : status; 2280} 2281 2282ssize_t 2283generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, 2284 unsigned long nr_segs, loff_t pos, loff_t *ppos, 2285 size_t count, ssize_t written) 2286{ 2287 struct file *file = iocb->ki_filp; 2288 ssize_t status; 2289 struct iov_iter i; 2290 2291 iov_iter_init(&i, iov, nr_segs, count, written); 2292 status = generic_perform_write(file, &i, pos); 2293 2294 if (likely(status >= 0)) { 2295 written += status; 2296 *ppos = pos + status; 2297 } 2298 2299 return written ? written : status; 2300} 2301EXPORT_SYMBOL(generic_file_buffered_write); 2302 2303/** 2304 * __generic_file_aio_write - write data to a file 2305 * @iocb: IO state structure (file, offset, etc.) 2306 * @iov: vector with data to write 2307 * @nr_segs: number of segments in the vector 2308 * @ppos: position where to write 2309 * 2310 * This function does all the work needed for actually writing data to a 2311 * file. It does all basic checks, removes SUID from the file, updates 2312 * modification times and calls proper subroutines depending on whether we 2313 * do direct IO or a standard buffered write. 2314 * 2315 * It expects i_mutex to be grabbed unless we work on a block device or similar 2316 * object which does not need locking at all. 2317 * 2318 * This function does *not* take care of syncing data in case of O_SYNC write. 2319 * A caller has to handle it. This is mainly due to the fact that we want to 2320 * avoid syncing under i_mutex. 2321 */ 2322ssize_t __generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov, 2323 unsigned long nr_segs, loff_t *ppos) 2324{ 2325 struct file *file = iocb->ki_filp; 2326 struct address_space * mapping = file->f_mapping; 2327 size_t ocount; /* original count */ 2328 size_t count; /* after file limit checks */ 2329 struct inode *inode = mapping->host; 2330 loff_t pos; 2331 ssize_t written; 2332 ssize_t err; 2333 2334 ocount = 0; 2335 err = generic_segment_checks(iov, &nr_segs, &ocount, VERIFY_READ); 2336 if (err) 2337 return err; 2338 2339 count = ocount; 2340 pos = *ppos; 2341 2342 vfs_check_frozen(inode->i_sb, SB_FREEZE_WRITE); 2343 2344 /* We can write back this queue in page reclaim */ 2345 current->backing_dev_info = mapping->backing_dev_info; 2346 written = 0; 2347 2348 err = generic_write_checks(file, &pos, &count, S_ISBLK(inode->i_mode)); 2349 if (err) 2350 goto out; 2351 2352 if (count == 0) 2353 goto out; 2354 2355 err = file_remove_suid(file); 2356 if (err) 2357 goto out; 2358 2359 file_update_time(file); 2360 2361 /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */ 2362 if (unlikely(file->f_flags & O_DIRECT)) { 2363 loff_t endbyte; 2364 ssize_t written_buffered; 2365 2366 written = generic_file_direct_write(iocb, iov, &nr_segs, pos, 2367 ppos, count, ocount); 2368 if (written < 0 || written == count) 2369 goto out; 2370 /* 2371 * direct-io write to a hole: fall through to buffered I/O 2372 * for completing the rest of the request. 2373 */ 2374 pos += written; 2375 count -= written; 2376 written_buffered = generic_file_buffered_write(iocb, iov, 2377 nr_segs, pos, ppos, count, 2378 written); 2379 /* 2380 * If generic_file_buffered_write() retuned a synchronous error 2381 * then we want to return the number of bytes which were 2382 * direct-written, or the error code if that was zero. Note 2383 * that this differs from normal direct-io semantics, which 2384 * will return -EFOO even if some bytes were written. 2385 */ 2386 if (written_buffered < 0) { 2387 err = written_buffered; 2388 goto out; 2389 } 2390 2391 /* 2392 * We need to ensure that the page cache pages are written to 2393 * disk and invalidated to preserve the expected O_DIRECT 2394 * semantics. 2395 */ 2396 endbyte = pos + written_buffered - written - 1; 2397 err = filemap_write_and_wait_range(file->f_mapping, pos, endbyte); 2398 if (err == 0) { 2399 written = written_buffered; 2400 invalidate_mapping_pages(mapping, 2401 pos >> PAGE_CACHE_SHIFT, 2402 endbyte >> PAGE_CACHE_SHIFT); 2403 } else { 2404 /* 2405 * We don't know how much we wrote, so just return 2406 * the number of bytes which were direct-written 2407 */ 2408 } 2409 } else { 2410 written = generic_file_buffered_write(iocb, iov, nr_segs, 2411 pos, ppos, count, written); 2412 } 2413out: 2414 current->backing_dev_info = NULL; 2415 return written ? written : err; 2416} 2417EXPORT_SYMBOL(__generic_file_aio_write); 2418 2419/** 2420 * generic_file_aio_write - write data to a file 2421 * @iocb: IO state structure 2422 * @iov: vector with data to write 2423 * @nr_segs: number of segments in the vector 2424 * @pos: position in file where to write 2425 * 2426 * This is a wrapper around __generic_file_aio_write() to be used by most 2427 * filesystems. It takes care of syncing the file in case of O_SYNC file 2428 * and acquires i_mutex as needed. 2429 */ 2430ssize_t generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov, 2431 unsigned long nr_segs, loff_t pos) 2432{ 2433 struct file *file = iocb->ki_filp; 2434 struct inode *inode = file->f_mapping->host; 2435 ssize_t ret; 2436 2437 BUG_ON(iocb->ki_pos != pos); 2438 2439 mutex_lock(&inode->i_mutex); 2440 ret = __generic_file_aio_write(iocb, iov, nr_segs, &iocb->ki_pos); 2441 mutex_unlock(&inode->i_mutex); 2442 2443 if (ret > 0 || ret == -EIOCBQUEUED) { 2444 ssize_t err; 2445 2446 err = generic_write_sync(file, pos, ret); 2447 if (err < 0 && ret > 0) 2448 ret = err; 2449 } 2450 return ret; 2451} 2452EXPORT_SYMBOL(generic_file_aio_write); 2453 2454/** 2455 * try_to_release_page() - release old fs-specific metadata on a page 2456 * 2457 * @page: the page which the kernel is trying to free 2458 * @gfp_mask: memory allocation flags (and I/O mode) 2459 * 2460 * The address_space is to try to release any data against the page 2461 * (presumably at page->private). If the release was successful, return `1'. 2462 * Otherwise return zero. 2463 * 2464 * This may also be called if PG_fscache is set on a page, indicating that the 2465 * page is known to the local caching routines. 2466 * 2467 * The @gfp_mask argument specifies whether I/O may be performed to release 2468 * this page (__GFP_IO), and whether the call may block (__GFP_WAIT & __GFP_FS). 2469 * 2470 */ 2471int try_to_release_page(struct page *page, gfp_t gfp_mask) 2472{ 2473 struct address_space * const mapping = page->mapping; 2474 2475 BUG_ON(!PageLocked(page)); 2476 if (PageWriteback(page)) 2477 return 0; 2478 2479 if (mapping && mapping->a_ops->releasepage) 2480 return mapping->a_ops->releasepage(page, gfp_mask); 2481 return try_to_free_buffers(page); 2482} 2483 2484EXPORT_SYMBOL(try_to_release_page); 2485