filemap.c revision 5307cc1aa53850f017c8053db034cf950b670ac9
1/* 2 * linux/mm/filemap.c 3 * 4 * Copyright (C) 1994-1999 Linus Torvalds 5 */ 6 7/* 8 * This file handles the generic file mmap semantics used by 9 * most "normal" filesystems (but you don't /have/ to use this: 10 * the NFS filesystem used to do this differently, for example) 11 */ 12#include <linux/module.h> 13#include <linux/slab.h> 14#include <linux/compiler.h> 15#include <linux/fs.h> 16#include <linux/uaccess.h> 17#include <linux/aio.h> 18#include <linux/capability.h> 19#include <linux/kernel_stat.h> 20#include <linux/mm.h> 21#include <linux/swap.h> 22#include <linux/mman.h> 23#include <linux/pagemap.h> 24#include <linux/file.h> 25#include <linux/uio.h> 26#include <linux/hash.h> 27#include <linux/writeback.h> 28#include <linux/backing-dev.h> 29#include <linux/pagevec.h> 30#include <linux/blkdev.h> 31#include <linux/backing-dev.h> 32#include <linux/security.h> 33#include <linux/syscalls.h> 34#include <linux/cpuset.h> 35#include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */ 36#include "internal.h" 37 38/* 39 * FIXME: remove all knowledge of the buffer layer from the core VM 40 */ 41#include <linux/buffer_head.h> /* for generic_osync_inode */ 42 43#include <asm/mman.h> 44 45static ssize_t 46generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, 47 loff_t offset, unsigned long nr_segs); 48 49/* 50 * Shared mappings implemented 30.11.1994. It's not fully working yet, 51 * though. 52 * 53 * Shared mappings now work. 15.8.1995 Bruno. 54 * 55 * finished 'unifying' the page and buffer cache and SMP-threaded the 56 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com> 57 * 58 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de> 59 */ 60 61/* 62 * Lock ordering: 63 * 64 * ->i_mmap_lock (vmtruncate) 65 * ->private_lock (__free_pte->__set_page_dirty_buffers) 66 * ->swap_lock (exclusive_swap_page, others) 67 * ->mapping->tree_lock 68 * ->zone.lock 69 * 70 * ->i_mutex 71 * ->i_mmap_lock (truncate->unmap_mapping_range) 72 * 73 * ->mmap_sem 74 * ->i_mmap_lock 75 * ->page_table_lock or pte_lock (various, mainly in memory.c) 76 * ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock) 77 * 78 * ->mmap_sem 79 * ->lock_page (access_process_vm) 80 * 81 * ->i_mutex (generic_file_buffered_write) 82 * ->mmap_sem (fault_in_pages_readable->do_page_fault) 83 * 84 * ->i_mutex 85 * ->i_alloc_sem (various) 86 * 87 * ->inode_lock 88 * ->sb_lock (fs/fs-writeback.c) 89 * ->mapping->tree_lock (__sync_single_inode) 90 * 91 * ->i_mmap_lock 92 * ->anon_vma.lock (vma_adjust) 93 * 94 * ->anon_vma.lock 95 * ->page_table_lock or pte_lock (anon_vma_prepare and various) 96 * 97 * ->page_table_lock or pte_lock 98 * ->swap_lock (try_to_unmap_one) 99 * ->private_lock (try_to_unmap_one) 100 * ->tree_lock (try_to_unmap_one) 101 * ->zone.lru_lock (follow_page->mark_page_accessed) 102 * ->zone.lru_lock (check_pte_range->isolate_lru_page) 103 * ->private_lock (page_remove_rmap->set_page_dirty) 104 * ->tree_lock (page_remove_rmap->set_page_dirty) 105 * ->inode_lock (page_remove_rmap->set_page_dirty) 106 * ->inode_lock (zap_pte_range->set_page_dirty) 107 * ->private_lock (zap_pte_range->__set_page_dirty_buffers) 108 * 109 * ->task->proc_lock 110 * ->dcache_lock (proc_pid_lookup) 111 */ 112 113/* 114 * Remove a page from the page cache and free it. Caller has to make 115 * sure the page is locked and that nobody else uses it - or that usage 116 * is safe. The caller must hold a write_lock on the mapping's tree_lock. 117 */ 118void __remove_from_page_cache(struct page *page) 119{ 120 struct address_space *mapping = page->mapping; 121 122 radix_tree_delete(&mapping->page_tree, page->index); 123 page->mapping = NULL; 124 mapping->nrpages--; 125 __dec_zone_page_state(page, NR_FILE_PAGES); 126 BUG_ON(page_mapped(page)); 127} 128 129void remove_from_page_cache(struct page *page) 130{ 131 struct address_space *mapping = page->mapping; 132 133 BUG_ON(!PageLocked(page)); 134 135 write_lock_irq(&mapping->tree_lock); 136 __remove_from_page_cache(page); 137 write_unlock_irq(&mapping->tree_lock); 138} 139 140static int sync_page(void *word) 141{ 142 struct address_space *mapping; 143 struct page *page; 144 145 page = container_of((unsigned long *)word, struct page, flags); 146 147 /* 148 * page_mapping() is being called without PG_locked held. 149 * Some knowledge of the state and use of the page is used to 150 * reduce the requirements down to a memory barrier. 151 * The danger here is of a stale page_mapping() return value 152 * indicating a struct address_space different from the one it's 153 * associated with when it is associated with one. 154 * After smp_mb(), it's either the correct page_mapping() for 155 * the page, or an old page_mapping() and the page's own 156 * page_mapping() has gone NULL. 157 * The ->sync_page() address_space operation must tolerate 158 * page_mapping() going NULL. By an amazing coincidence, 159 * this comes about because none of the users of the page 160 * in the ->sync_page() methods make essential use of the 161 * page_mapping(), merely passing the page down to the backing 162 * device's unplug functions when it's non-NULL, which in turn 163 * ignore it for all cases but swap, where only page_private(page) is 164 * of interest. When page_mapping() does go NULL, the entire 165 * call stack gracefully ignores the page and returns. 166 * -- wli 167 */ 168 smp_mb(); 169 mapping = page_mapping(page); 170 if (mapping && mapping->a_ops && mapping->a_ops->sync_page) 171 mapping->a_ops->sync_page(page); 172 io_schedule(); 173 return 0; 174} 175 176/** 177 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range 178 * @mapping: address space structure to write 179 * @start: offset in bytes where the range starts 180 * @end: offset in bytes where the range ends (inclusive) 181 * @sync_mode: enable synchronous operation 182 * 183 * Start writeback against all of a mapping's dirty pages that lie 184 * within the byte offsets <start, end> inclusive. 185 * 186 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as 187 * opposed to a regular memory cleansing writeback. The difference between 188 * these two operations is that if a dirty page/buffer is encountered, it must 189 * be waited upon, and not just skipped over. 190 */ 191int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start, 192 loff_t end, int sync_mode) 193{ 194 int ret; 195 struct writeback_control wbc = { 196 .sync_mode = sync_mode, 197 .nr_to_write = mapping->nrpages * 2, 198 .range_start = start, 199 .range_end = end, 200 }; 201 202 if (!mapping_cap_writeback_dirty(mapping)) 203 return 0; 204 205 ret = do_writepages(mapping, &wbc); 206 return ret; 207} 208 209static inline int __filemap_fdatawrite(struct address_space *mapping, 210 int sync_mode) 211{ 212 return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode); 213} 214 215int filemap_fdatawrite(struct address_space *mapping) 216{ 217 return __filemap_fdatawrite(mapping, WB_SYNC_ALL); 218} 219EXPORT_SYMBOL(filemap_fdatawrite); 220 221static int filemap_fdatawrite_range(struct address_space *mapping, loff_t start, 222 loff_t end) 223{ 224 return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL); 225} 226 227/** 228 * filemap_flush - mostly a non-blocking flush 229 * @mapping: target address_space 230 * 231 * This is a mostly non-blocking flush. Not suitable for data-integrity 232 * purposes - I/O may not be started against all dirty pages. 233 */ 234int filemap_flush(struct address_space *mapping) 235{ 236 return __filemap_fdatawrite(mapping, WB_SYNC_NONE); 237} 238EXPORT_SYMBOL(filemap_flush); 239 240/** 241 * wait_on_page_writeback_range - wait for writeback to complete 242 * @mapping: target address_space 243 * @start: beginning page index 244 * @end: ending page index 245 * 246 * Wait for writeback to complete against pages indexed by start->end 247 * inclusive 248 */ 249int wait_on_page_writeback_range(struct address_space *mapping, 250 pgoff_t start, pgoff_t end) 251{ 252 struct pagevec pvec; 253 int nr_pages; 254 int ret = 0; 255 pgoff_t index; 256 257 if (end < start) 258 return 0; 259 260 pagevec_init(&pvec, 0); 261 index = start; 262 while ((index <= end) && 263 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, 264 PAGECACHE_TAG_WRITEBACK, 265 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1)) != 0) { 266 unsigned i; 267 268 for (i = 0; i < nr_pages; i++) { 269 struct page *page = pvec.pages[i]; 270 271 /* until radix tree lookup accepts end_index */ 272 if (page->index > end) 273 continue; 274 275 wait_on_page_writeback(page); 276 if (PageError(page)) 277 ret = -EIO; 278 } 279 pagevec_release(&pvec); 280 cond_resched(); 281 } 282 283 /* Check for outstanding write errors */ 284 if (test_and_clear_bit(AS_ENOSPC, &mapping->flags)) 285 ret = -ENOSPC; 286 if (test_and_clear_bit(AS_EIO, &mapping->flags)) 287 ret = -EIO; 288 289 return ret; 290} 291 292/** 293 * sync_page_range - write and wait on all pages in the passed range 294 * @inode: target inode 295 * @mapping: target address_space 296 * @pos: beginning offset in pages to write 297 * @count: number of bytes to write 298 * 299 * Write and wait upon all the pages in the passed range. This is a "data 300 * integrity" operation. It waits upon in-flight writeout before starting and 301 * waiting upon new writeout. If there was an IO error, return it. 302 * 303 * We need to re-take i_mutex during the generic_osync_inode list walk because 304 * it is otherwise livelockable. 305 */ 306int sync_page_range(struct inode *inode, struct address_space *mapping, 307 loff_t pos, loff_t count) 308{ 309 pgoff_t start = pos >> PAGE_CACHE_SHIFT; 310 pgoff_t end = (pos + count - 1) >> PAGE_CACHE_SHIFT; 311 int ret; 312 313 if (!mapping_cap_writeback_dirty(mapping) || !count) 314 return 0; 315 ret = filemap_fdatawrite_range(mapping, pos, pos + count - 1); 316 if (ret == 0) { 317 mutex_lock(&inode->i_mutex); 318 ret = generic_osync_inode(inode, mapping, OSYNC_METADATA); 319 mutex_unlock(&inode->i_mutex); 320 } 321 if (ret == 0) 322 ret = wait_on_page_writeback_range(mapping, start, end); 323 return ret; 324} 325EXPORT_SYMBOL(sync_page_range); 326 327/** 328 * sync_page_range_nolock 329 * @inode: target inode 330 * @mapping: target address_space 331 * @pos: beginning offset in pages to write 332 * @count: number of bytes to write 333 * 334 * Note: Holding i_mutex across sync_page_range_nolock() is not a good idea 335 * as it forces O_SYNC writers to different parts of the same file 336 * to be serialised right until io completion. 337 */ 338int sync_page_range_nolock(struct inode *inode, struct address_space *mapping, 339 loff_t pos, loff_t count) 340{ 341 pgoff_t start = pos >> PAGE_CACHE_SHIFT; 342 pgoff_t end = (pos + count - 1) >> PAGE_CACHE_SHIFT; 343 int ret; 344 345 if (!mapping_cap_writeback_dirty(mapping) || !count) 346 return 0; 347 ret = filemap_fdatawrite_range(mapping, pos, pos + count - 1); 348 if (ret == 0) 349 ret = generic_osync_inode(inode, mapping, OSYNC_METADATA); 350 if (ret == 0) 351 ret = wait_on_page_writeback_range(mapping, start, end); 352 return ret; 353} 354EXPORT_SYMBOL(sync_page_range_nolock); 355 356/** 357 * filemap_fdatawait - wait for all under-writeback pages to complete 358 * @mapping: address space structure to wait for 359 * 360 * Walk the list of under-writeback pages of the given address space 361 * and wait for all of them. 362 */ 363int filemap_fdatawait(struct address_space *mapping) 364{ 365 loff_t i_size = i_size_read(mapping->host); 366 367 if (i_size == 0) 368 return 0; 369 370 return wait_on_page_writeback_range(mapping, 0, 371 (i_size - 1) >> PAGE_CACHE_SHIFT); 372} 373EXPORT_SYMBOL(filemap_fdatawait); 374 375int filemap_write_and_wait(struct address_space *mapping) 376{ 377 int err = 0; 378 379 if (mapping->nrpages) { 380 err = filemap_fdatawrite(mapping); 381 /* 382 * Even if the above returned error, the pages may be 383 * written partially (e.g. -ENOSPC), so we wait for it. 384 * But the -EIO is special case, it may indicate the worst 385 * thing (e.g. bug) happened, so we avoid waiting for it. 386 */ 387 if (err != -EIO) { 388 int err2 = filemap_fdatawait(mapping); 389 if (!err) 390 err = err2; 391 } 392 } 393 return err; 394} 395EXPORT_SYMBOL(filemap_write_and_wait); 396 397/** 398 * filemap_write_and_wait_range - write out & wait on a file range 399 * @mapping: the address_space for the pages 400 * @lstart: offset in bytes where the range starts 401 * @lend: offset in bytes where the range ends (inclusive) 402 * 403 * Write out and wait upon file offsets lstart->lend, inclusive. 404 * 405 * Note that `lend' is inclusive (describes the last byte to be written) so 406 * that this function can be used to write to the very end-of-file (end = -1). 407 */ 408int filemap_write_and_wait_range(struct address_space *mapping, 409 loff_t lstart, loff_t lend) 410{ 411 int err = 0; 412 413 if (mapping->nrpages) { 414 err = __filemap_fdatawrite_range(mapping, lstart, lend, 415 WB_SYNC_ALL); 416 /* See comment of filemap_write_and_wait() */ 417 if (err != -EIO) { 418 int err2 = wait_on_page_writeback_range(mapping, 419 lstart >> PAGE_CACHE_SHIFT, 420 lend >> PAGE_CACHE_SHIFT); 421 if (!err) 422 err = err2; 423 } 424 } 425 return err; 426} 427 428/** 429 * add_to_page_cache - add newly allocated pagecache pages 430 * @page: page to add 431 * @mapping: the page's address_space 432 * @offset: page index 433 * @gfp_mask: page allocation mode 434 * 435 * This function is used to add newly allocated pagecache pages; 436 * the page is new, so we can just run SetPageLocked() against it. 437 * The other page state flags were set by rmqueue(). 438 * 439 * This function does not add the page to the LRU. The caller must do that. 440 */ 441int add_to_page_cache(struct page *page, struct address_space *mapping, 442 pgoff_t offset, gfp_t gfp_mask) 443{ 444 int error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM); 445 446 if (error == 0) { 447 write_lock_irq(&mapping->tree_lock); 448 error = radix_tree_insert(&mapping->page_tree, offset, page); 449 if (!error) { 450 page_cache_get(page); 451 SetPageLocked(page); 452 page->mapping = mapping; 453 page->index = offset; 454 mapping->nrpages++; 455 __inc_zone_page_state(page, NR_FILE_PAGES); 456 } 457 write_unlock_irq(&mapping->tree_lock); 458 radix_tree_preload_end(); 459 } 460 return error; 461} 462EXPORT_SYMBOL(add_to_page_cache); 463 464int add_to_page_cache_lru(struct page *page, struct address_space *mapping, 465 pgoff_t offset, gfp_t gfp_mask) 466{ 467 int ret = add_to_page_cache(page, mapping, offset, gfp_mask); 468 if (ret == 0) 469 lru_cache_add(page); 470 return ret; 471} 472 473#ifdef CONFIG_NUMA 474struct page *__page_cache_alloc(gfp_t gfp) 475{ 476 if (cpuset_do_page_mem_spread()) { 477 int n = cpuset_mem_spread_node(); 478 return alloc_pages_node(n, gfp, 0); 479 } 480 return alloc_pages(gfp, 0); 481} 482EXPORT_SYMBOL(__page_cache_alloc); 483#endif 484 485static int __sleep_on_page_lock(void *word) 486{ 487 io_schedule(); 488 return 0; 489} 490 491/* 492 * In order to wait for pages to become available there must be 493 * waitqueues associated with pages. By using a hash table of 494 * waitqueues where the bucket discipline is to maintain all 495 * waiters on the same queue and wake all when any of the pages 496 * become available, and for the woken contexts to check to be 497 * sure the appropriate page became available, this saves space 498 * at a cost of "thundering herd" phenomena during rare hash 499 * collisions. 500 */ 501static wait_queue_head_t *page_waitqueue(struct page *page) 502{ 503 const struct zone *zone = page_zone(page); 504 505 return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)]; 506} 507 508static inline void wake_up_page(struct page *page, int bit) 509{ 510 __wake_up_bit(page_waitqueue(page), &page->flags, bit); 511} 512 513void fastcall wait_on_page_bit(struct page *page, int bit_nr) 514{ 515 DEFINE_WAIT_BIT(wait, &page->flags, bit_nr); 516 517 if (test_bit(bit_nr, &page->flags)) 518 __wait_on_bit(page_waitqueue(page), &wait, sync_page, 519 TASK_UNINTERRUPTIBLE); 520} 521EXPORT_SYMBOL(wait_on_page_bit); 522 523/** 524 * unlock_page - unlock a locked page 525 * @page: the page 526 * 527 * Unlocks the page and wakes up sleepers in ___wait_on_page_locked(). 528 * Also wakes sleepers in wait_on_page_writeback() because the wakeup 529 * mechananism between PageLocked pages and PageWriteback pages is shared. 530 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep. 531 * 532 * The first mb is necessary to safely close the critical section opened by the 533 * TestSetPageLocked(), the second mb is necessary to enforce ordering between 534 * the clear_bit and the read of the waitqueue (to avoid SMP races with a 535 * parallel wait_on_page_locked()). 536 */ 537void fastcall unlock_page(struct page *page) 538{ 539 smp_mb__before_clear_bit(); 540 if (!TestClearPageLocked(page)) 541 BUG(); 542 smp_mb__after_clear_bit(); 543 wake_up_page(page, PG_locked); 544} 545EXPORT_SYMBOL(unlock_page); 546 547/** 548 * end_page_writeback - end writeback against a page 549 * @page: the page 550 */ 551void end_page_writeback(struct page *page) 552{ 553 if (!TestClearPageReclaim(page) || rotate_reclaimable_page(page)) { 554 if (!test_clear_page_writeback(page)) 555 BUG(); 556 } 557 smp_mb__after_clear_bit(); 558 wake_up_page(page, PG_writeback); 559} 560EXPORT_SYMBOL(end_page_writeback); 561 562/** 563 * __lock_page - get a lock on the page, assuming we need to sleep to get it 564 * @page: the page to lock 565 * 566 * Ugly. Running sync_page() in state TASK_UNINTERRUPTIBLE is scary. If some 567 * random driver's requestfn sets TASK_RUNNING, we could busywait. However 568 * chances are that on the second loop, the block layer's plug list is empty, 569 * so sync_page() will then return in state TASK_UNINTERRUPTIBLE. 570 */ 571void fastcall __lock_page(struct page *page) 572{ 573 DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); 574 575 __wait_on_bit_lock(page_waitqueue(page), &wait, sync_page, 576 TASK_UNINTERRUPTIBLE); 577} 578EXPORT_SYMBOL(__lock_page); 579 580/* 581 * Variant of lock_page that does not require the caller to hold a reference 582 * on the page's mapping. 583 */ 584void fastcall __lock_page_nosync(struct page *page) 585{ 586 DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); 587 __wait_on_bit_lock(page_waitqueue(page), &wait, __sleep_on_page_lock, 588 TASK_UNINTERRUPTIBLE); 589} 590 591/** 592 * find_get_page - find and get a page reference 593 * @mapping: the address_space to search 594 * @offset: the page index 595 * 596 * Is there a pagecache struct page at the given (mapping, offset) tuple? 597 * If yes, increment its refcount and return it; if no, return NULL. 598 */ 599struct page * find_get_page(struct address_space *mapping, pgoff_t offset) 600{ 601 struct page *page; 602 603 read_lock_irq(&mapping->tree_lock); 604 page = radix_tree_lookup(&mapping->page_tree, offset); 605 if (page) 606 page_cache_get(page); 607 read_unlock_irq(&mapping->tree_lock); 608 return page; 609} 610EXPORT_SYMBOL(find_get_page); 611 612/** 613 * find_lock_page - locate, pin and lock a pagecache page 614 * @mapping: the address_space to search 615 * @offset: the page index 616 * 617 * Locates the desired pagecache page, locks it, increments its reference 618 * count and returns its address. 619 * 620 * Returns zero if the page was not present. find_lock_page() may sleep. 621 */ 622struct page *find_lock_page(struct address_space *mapping, 623 pgoff_t offset) 624{ 625 struct page *page; 626 627repeat: 628 read_lock_irq(&mapping->tree_lock); 629 page = radix_tree_lookup(&mapping->page_tree, offset); 630 if (page) { 631 page_cache_get(page); 632 if (TestSetPageLocked(page)) { 633 read_unlock_irq(&mapping->tree_lock); 634 __lock_page(page); 635 636 /* Has the page been truncated while we slept? */ 637 if (unlikely(page->mapping != mapping)) { 638 unlock_page(page); 639 page_cache_release(page); 640 goto repeat; 641 } 642 VM_BUG_ON(page->index != offset); 643 goto out; 644 } 645 } 646 read_unlock_irq(&mapping->tree_lock); 647out: 648 return page; 649} 650EXPORT_SYMBOL(find_lock_page); 651 652/** 653 * find_or_create_page - locate or add a pagecache page 654 * @mapping: the page's address_space 655 * @index: the page's index into the mapping 656 * @gfp_mask: page allocation mode 657 * 658 * Locates a page in the pagecache. If the page is not present, a new page 659 * is allocated using @gfp_mask and is added to the pagecache and to the VM's 660 * LRU list. The returned page is locked and has its reference count 661 * incremented. 662 * 663 * find_or_create_page() may sleep, even if @gfp_flags specifies an atomic 664 * allocation! 665 * 666 * find_or_create_page() returns the desired page's address, or zero on 667 * memory exhaustion. 668 */ 669struct page *find_or_create_page(struct address_space *mapping, 670 pgoff_t index, gfp_t gfp_mask) 671{ 672 struct page *page; 673 int err; 674repeat: 675 page = find_lock_page(mapping, index); 676 if (!page) { 677 page = __page_cache_alloc(gfp_mask); 678 if (!page) 679 return NULL; 680 err = add_to_page_cache_lru(page, mapping, index, gfp_mask); 681 if (unlikely(err)) { 682 page_cache_release(page); 683 page = NULL; 684 if (err == -EEXIST) 685 goto repeat; 686 } 687 } 688 return page; 689} 690EXPORT_SYMBOL(find_or_create_page); 691 692/** 693 * find_get_pages - gang pagecache lookup 694 * @mapping: The address_space to search 695 * @start: The starting page index 696 * @nr_pages: The maximum number of pages 697 * @pages: Where the resulting pages are placed 698 * 699 * find_get_pages() will search for and return a group of up to 700 * @nr_pages pages in the mapping. The pages are placed at @pages. 701 * find_get_pages() takes a reference against the returned pages. 702 * 703 * The search returns a group of mapping-contiguous pages with ascending 704 * indexes. There may be holes in the indices due to not-present pages. 705 * 706 * find_get_pages() returns the number of pages which were found. 707 */ 708unsigned find_get_pages(struct address_space *mapping, pgoff_t start, 709 unsigned int nr_pages, struct page **pages) 710{ 711 unsigned int i; 712 unsigned int ret; 713 714 read_lock_irq(&mapping->tree_lock); 715 ret = radix_tree_gang_lookup(&mapping->page_tree, 716 (void **)pages, start, nr_pages); 717 for (i = 0; i < ret; i++) 718 page_cache_get(pages[i]); 719 read_unlock_irq(&mapping->tree_lock); 720 return ret; 721} 722 723/** 724 * find_get_pages_contig - gang contiguous pagecache lookup 725 * @mapping: The address_space to search 726 * @index: The starting page index 727 * @nr_pages: The maximum number of pages 728 * @pages: Where the resulting pages are placed 729 * 730 * find_get_pages_contig() works exactly like find_get_pages(), except 731 * that the returned number of pages are guaranteed to be contiguous. 732 * 733 * find_get_pages_contig() returns the number of pages which were found. 734 */ 735unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, 736 unsigned int nr_pages, struct page **pages) 737{ 738 unsigned int i; 739 unsigned int ret; 740 741 read_lock_irq(&mapping->tree_lock); 742 ret = radix_tree_gang_lookup(&mapping->page_tree, 743 (void **)pages, index, nr_pages); 744 for (i = 0; i < ret; i++) { 745 if (pages[i]->mapping == NULL || pages[i]->index != index) 746 break; 747 748 page_cache_get(pages[i]); 749 index++; 750 } 751 read_unlock_irq(&mapping->tree_lock); 752 return i; 753} 754EXPORT_SYMBOL(find_get_pages_contig); 755 756/** 757 * find_get_pages_tag - find and return pages that match @tag 758 * @mapping: the address_space to search 759 * @index: the starting page index 760 * @tag: the tag index 761 * @nr_pages: the maximum number of pages 762 * @pages: where the resulting pages are placed 763 * 764 * Like find_get_pages, except we only return pages which are tagged with 765 * @tag. We update @index to index the next page for the traversal. 766 */ 767unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index, 768 int tag, unsigned int nr_pages, struct page **pages) 769{ 770 unsigned int i; 771 unsigned int ret; 772 773 read_lock_irq(&mapping->tree_lock); 774 ret = radix_tree_gang_lookup_tag(&mapping->page_tree, 775 (void **)pages, *index, nr_pages, tag); 776 for (i = 0; i < ret; i++) 777 page_cache_get(pages[i]); 778 if (ret) 779 *index = pages[ret - 1]->index + 1; 780 read_unlock_irq(&mapping->tree_lock); 781 return ret; 782} 783EXPORT_SYMBOL(find_get_pages_tag); 784 785/** 786 * grab_cache_page_nowait - returns locked page at given index in given cache 787 * @mapping: target address_space 788 * @index: the page index 789 * 790 * Same as grab_cache_page(), but do not wait if the page is unavailable. 791 * This is intended for speculative data generators, where the data can 792 * be regenerated if the page couldn't be grabbed. This routine should 793 * be safe to call while holding the lock for another page. 794 * 795 * Clear __GFP_FS when allocating the page to avoid recursion into the fs 796 * and deadlock against the caller's locked page. 797 */ 798struct page * 799grab_cache_page_nowait(struct address_space *mapping, pgoff_t index) 800{ 801 struct page *page = find_get_page(mapping, index); 802 803 if (page) { 804 if (!TestSetPageLocked(page)) 805 return page; 806 page_cache_release(page); 807 return NULL; 808 } 809 page = __page_cache_alloc(mapping_gfp_mask(mapping) & ~__GFP_FS); 810 if (page && add_to_page_cache_lru(page, mapping, index, GFP_KERNEL)) { 811 page_cache_release(page); 812 page = NULL; 813 } 814 return page; 815} 816EXPORT_SYMBOL(grab_cache_page_nowait); 817 818/* 819 * CD/DVDs are error prone. When a medium error occurs, the driver may fail 820 * a _large_ part of the i/o request. Imagine the worst scenario: 821 * 822 * ---R__________________________________________B__________ 823 * ^ reading here ^ bad block(assume 4k) 824 * 825 * read(R) => miss => readahead(R...B) => media error => frustrating retries 826 * => failing the whole request => read(R) => read(R+1) => 827 * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) => 828 * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) => 829 * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ...... 830 * 831 * It is going insane. Fix it by quickly scaling down the readahead size. 832 */ 833static void shrink_readahead_size_eio(struct file *filp, 834 struct file_ra_state *ra) 835{ 836 if (!ra->ra_pages) 837 return; 838 839 ra->ra_pages /= 4; 840} 841 842/** 843 * do_generic_mapping_read - generic file read routine 844 * @mapping: address_space to be read 845 * @ra: file's readahead state 846 * @filp: the file to read 847 * @ppos: current file position 848 * @desc: read_descriptor 849 * @actor: read method 850 * 851 * This is a generic file read routine, and uses the 852 * mapping->a_ops->readpage() function for the actual low-level stuff. 853 * 854 * This is really ugly. But the goto's actually try to clarify some 855 * of the logic when it comes to error handling etc. 856 * 857 * Note the struct file* is only passed for the use of readpage. 858 * It may be NULL. 859 */ 860void do_generic_mapping_read(struct address_space *mapping, 861 struct file_ra_state *ra, 862 struct file *filp, 863 loff_t *ppos, 864 read_descriptor_t *desc, 865 read_actor_t actor) 866{ 867 struct inode *inode = mapping->host; 868 pgoff_t index; 869 pgoff_t last_index; 870 pgoff_t prev_index; 871 unsigned long offset; /* offset into pagecache page */ 872 unsigned int prev_offset; 873 int error; 874 875 index = *ppos >> PAGE_CACHE_SHIFT; 876 prev_index = ra->prev_pos >> PAGE_CACHE_SHIFT; 877 prev_offset = ra->prev_pos & (PAGE_CACHE_SIZE-1); 878 last_index = (*ppos + desc->count + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT; 879 offset = *ppos & ~PAGE_CACHE_MASK; 880 881 for (;;) { 882 struct page *page; 883 pgoff_t end_index; 884 loff_t isize; 885 unsigned long nr, ret; 886 887 cond_resched(); 888find_page: 889 page = find_get_page(mapping, index); 890 if (!page) { 891 page_cache_sync_readahead(mapping, 892 ra, filp, 893 index, last_index - index); 894 page = find_get_page(mapping, index); 895 if (unlikely(page == NULL)) 896 goto no_cached_page; 897 } 898 if (PageReadahead(page)) { 899 page_cache_async_readahead(mapping, 900 ra, filp, page, 901 index, last_index - index); 902 } 903 if (!PageUptodate(page)) 904 goto page_not_up_to_date; 905page_ok: 906 /* 907 * i_size must be checked after we know the page is Uptodate. 908 * 909 * Checking i_size after the check allows us to calculate 910 * the correct value for "nr", which means the zero-filled 911 * part of the page is not copied back to userspace (unless 912 * another truncate extends the file - this is desired though). 913 */ 914 915 isize = i_size_read(inode); 916 end_index = (isize - 1) >> PAGE_CACHE_SHIFT; 917 if (unlikely(!isize || index > end_index)) { 918 page_cache_release(page); 919 goto out; 920 } 921 922 /* nr is the maximum number of bytes to copy from this page */ 923 nr = PAGE_CACHE_SIZE; 924 if (index == end_index) { 925 nr = ((isize - 1) & ~PAGE_CACHE_MASK) + 1; 926 if (nr <= offset) { 927 page_cache_release(page); 928 goto out; 929 } 930 } 931 nr = nr - offset; 932 933 /* If users can be writing to this page using arbitrary 934 * virtual addresses, take care about potential aliasing 935 * before reading the page on the kernel side. 936 */ 937 if (mapping_writably_mapped(mapping)) 938 flush_dcache_page(page); 939 940 /* 941 * When a sequential read accesses a page several times, 942 * only mark it as accessed the first time. 943 */ 944 if (prev_index != index || offset != prev_offset) 945 mark_page_accessed(page); 946 prev_index = index; 947 948 /* 949 * Ok, we have the page, and it's up-to-date, so 950 * now we can copy it to user space... 951 * 952 * The actor routine returns how many bytes were actually used.. 953 * NOTE! This may not be the same as how much of a user buffer 954 * we filled up (we may be padding etc), so we can only update 955 * "pos" here (the actor routine has to update the user buffer 956 * pointers and the remaining count). 957 */ 958 ret = actor(desc, page, offset, nr); 959 offset += ret; 960 index += offset >> PAGE_CACHE_SHIFT; 961 offset &= ~PAGE_CACHE_MASK; 962 prev_offset = offset; 963 964 page_cache_release(page); 965 if (ret == nr && desc->count) 966 continue; 967 goto out; 968 969page_not_up_to_date: 970 /* Get exclusive access to the page ... */ 971 lock_page(page); 972 973 /* Did it get truncated before we got the lock? */ 974 if (!page->mapping) { 975 unlock_page(page); 976 page_cache_release(page); 977 continue; 978 } 979 980 /* Did somebody else fill it already? */ 981 if (PageUptodate(page)) { 982 unlock_page(page); 983 goto page_ok; 984 } 985 986readpage: 987 /* Start the actual read. The read will unlock the page. */ 988 error = mapping->a_ops->readpage(filp, page); 989 990 if (unlikely(error)) { 991 if (error == AOP_TRUNCATED_PAGE) { 992 page_cache_release(page); 993 goto find_page; 994 } 995 goto readpage_error; 996 } 997 998 if (!PageUptodate(page)) { 999 lock_page(page); 1000 if (!PageUptodate(page)) { 1001 if (page->mapping == NULL) { 1002 /* 1003 * invalidate_inode_pages got it 1004 */ 1005 unlock_page(page); 1006 page_cache_release(page); 1007 goto find_page; 1008 } 1009 unlock_page(page); 1010 error = -EIO; 1011 shrink_readahead_size_eio(filp, ra); 1012 goto readpage_error; 1013 } 1014 unlock_page(page); 1015 } 1016 1017 goto page_ok; 1018 1019readpage_error: 1020 /* UHHUH! A synchronous read error occurred. Report it */ 1021 desc->error = error; 1022 page_cache_release(page); 1023 goto out; 1024 1025no_cached_page: 1026 /* 1027 * Ok, it wasn't cached, so we need to create a new 1028 * page.. 1029 */ 1030 page = page_cache_alloc_cold(mapping); 1031 if (!page) { 1032 desc->error = -ENOMEM; 1033 goto out; 1034 } 1035 error = add_to_page_cache_lru(page, mapping, 1036 index, GFP_KERNEL); 1037 if (error) { 1038 page_cache_release(page); 1039 if (error == -EEXIST) 1040 goto find_page; 1041 desc->error = error; 1042 goto out; 1043 } 1044 goto readpage; 1045 } 1046 1047out: 1048 ra->prev_pos = prev_index; 1049 ra->prev_pos <<= PAGE_CACHE_SHIFT; 1050 ra->prev_pos |= prev_offset; 1051 1052 *ppos = ((loff_t)index << PAGE_CACHE_SHIFT) + offset; 1053 if (filp) 1054 file_accessed(filp); 1055} 1056EXPORT_SYMBOL(do_generic_mapping_read); 1057 1058int file_read_actor(read_descriptor_t *desc, struct page *page, 1059 unsigned long offset, unsigned long size) 1060{ 1061 char *kaddr; 1062 unsigned long left, count = desc->count; 1063 1064 if (size > count) 1065 size = count; 1066 1067 /* 1068 * Faults on the destination of a read are common, so do it before 1069 * taking the kmap. 1070 */ 1071 if (!fault_in_pages_writeable(desc->arg.buf, size)) { 1072 kaddr = kmap_atomic(page, KM_USER0); 1073 left = __copy_to_user_inatomic(desc->arg.buf, 1074 kaddr + offset, size); 1075 kunmap_atomic(kaddr, KM_USER0); 1076 if (left == 0) 1077 goto success; 1078 } 1079 1080 /* Do it the slow way */ 1081 kaddr = kmap(page); 1082 left = __copy_to_user(desc->arg.buf, kaddr + offset, size); 1083 kunmap(page); 1084 1085 if (left) { 1086 size -= left; 1087 desc->error = -EFAULT; 1088 } 1089success: 1090 desc->count = count - size; 1091 desc->written += size; 1092 desc->arg.buf += size; 1093 return size; 1094} 1095 1096/* 1097 * Performs necessary checks before doing a write 1098 * @iov: io vector request 1099 * @nr_segs: number of segments in the iovec 1100 * @count: number of bytes to write 1101 * @access_flags: type of access: %VERIFY_READ or %VERIFY_WRITE 1102 * 1103 * Adjust number of segments and amount of bytes to write (nr_segs should be 1104 * properly initialized first). Returns appropriate error code that caller 1105 * should return or zero in case that write should be allowed. 1106 */ 1107int generic_segment_checks(const struct iovec *iov, 1108 unsigned long *nr_segs, size_t *count, int access_flags) 1109{ 1110 unsigned long seg; 1111 size_t cnt = 0; 1112 for (seg = 0; seg < *nr_segs; seg++) { 1113 const struct iovec *iv = &iov[seg]; 1114 1115 /* 1116 * If any segment has a negative length, or the cumulative 1117 * length ever wraps negative then return -EINVAL. 1118 */ 1119 cnt += iv->iov_len; 1120 if (unlikely((ssize_t)(cnt|iv->iov_len) < 0)) 1121 return -EINVAL; 1122 if (access_ok(access_flags, iv->iov_base, iv->iov_len)) 1123 continue; 1124 if (seg == 0) 1125 return -EFAULT; 1126 *nr_segs = seg; 1127 cnt -= iv->iov_len; /* This segment is no good */ 1128 break; 1129 } 1130 *count = cnt; 1131 return 0; 1132} 1133EXPORT_SYMBOL(generic_segment_checks); 1134 1135/** 1136 * generic_file_aio_read - generic filesystem read routine 1137 * @iocb: kernel I/O control block 1138 * @iov: io vector request 1139 * @nr_segs: number of segments in the iovec 1140 * @pos: current file position 1141 * 1142 * This is the "read()" routine for all filesystems 1143 * that can use the page cache directly. 1144 */ 1145ssize_t 1146generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov, 1147 unsigned long nr_segs, loff_t pos) 1148{ 1149 struct file *filp = iocb->ki_filp; 1150 ssize_t retval; 1151 unsigned long seg; 1152 size_t count; 1153 loff_t *ppos = &iocb->ki_pos; 1154 1155 count = 0; 1156 retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE); 1157 if (retval) 1158 return retval; 1159 1160 /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */ 1161 if (filp->f_flags & O_DIRECT) { 1162 loff_t size; 1163 struct address_space *mapping; 1164 struct inode *inode; 1165 1166 mapping = filp->f_mapping; 1167 inode = mapping->host; 1168 retval = 0; 1169 if (!count) 1170 goto out; /* skip atime */ 1171 size = i_size_read(inode); 1172 if (pos < size) { 1173 retval = generic_file_direct_IO(READ, iocb, 1174 iov, pos, nr_segs); 1175 if (retval > 0) 1176 *ppos = pos + retval; 1177 } 1178 if (likely(retval != 0)) { 1179 file_accessed(filp); 1180 goto out; 1181 } 1182 } 1183 1184 retval = 0; 1185 if (count) { 1186 for (seg = 0; seg < nr_segs; seg++) { 1187 read_descriptor_t desc; 1188 1189 desc.written = 0; 1190 desc.arg.buf = iov[seg].iov_base; 1191 desc.count = iov[seg].iov_len; 1192 if (desc.count == 0) 1193 continue; 1194 desc.error = 0; 1195 do_generic_file_read(filp,ppos,&desc,file_read_actor); 1196 retval += desc.written; 1197 if (desc.error) { 1198 retval = retval ?: desc.error; 1199 break; 1200 } 1201 if (desc.count > 0) 1202 break; 1203 } 1204 } 1205out: 1206 return retval; 1207} 1208EXPORT_SYMBOL(generic_file_aio_read); 1209 1210static ssize_t 1211do_readahead(struct address_space *mapping, struct file *filp, 1212 pgoff_t index, unsigned long nr) 1213{ 1214 if (!mapping || !mapping->a_ops || !mapping->a_ops->readpage) 1215 return -EINVAL; 1216 1217 force_page_cache_readahead(mapping, filp, index, 1218 max_sane_readahead(nr)); 1219 return 0; 1220} 1221 1222asmlinkage ssize_t sys_readahead(int fd, loff_t offset, size_t count) 1223{ 1224 ssize_t ret; 1225 struct file *file; 1226 1227 ret = -EBADF; 1228 file = fget(fd); 1229 if (file) { 1230 if (file->f_mode & FMODE_READ) { 1231 struct address_space *mapping = file->f_mapping; 1232 pgoff_t start = offset >> PAGE_CACHE_SHIFT; 1233 pgoff_t end = (offset + count - 1) >> PAGE_CACHE_SHIFT; 1234 unsigned long len = end - start + 1; 1235 ret = do_readahead(mapping, file, start, len); 1236 } 1237 fput(file); 1238 } 1239 return ret; 1240} 1241 1242#ifdef CONFIG_MMU 1243/** 1244 * page_cache_read - adds requested page to the page cache if not already there 1245 * @file: file to read 1246 * @offset: page index 1247 * 1248 * This adds the requested page to the page cache if it isn't already there, 1249 * and schedules an I/O to read in its contents from disk. 1250 */ 1251static int fastcall page_cache_read(struct file * file, pgoff_t offset) 1252{ 1253 struct address_space *mapping = file->f_mapping; 1254 struct page *page; 1255 int ret; 1256 1257 do { 1258 page = page_cache_alloc_cold(mapping); 1259 if (!page) 1260 return -ENOMEM; 1261 1262 ret = add_to_page_cache_lru(page, mapping, offset, GFP_KERNEL); 1263 if (ret == 0) 1264 ret = mapping->a_ops->readpage(file, page); 1265 else if (ret == -EEXIST) 1266 ret = 0; /* losing race to add is OK */ 1267 1268 page_cache_release(page); 1269 1270 } while (ret == AOP_TRUNCATED_PAGE); 1271 1272 return ret; 1273} 1274 1275#define MMAP_LOTSAMISS (100) 1276 1277/** 1278 * filemap_fault - read in file data for page fault handling 1279 * @vma: vma in which the fault was taken 1280 * @vmf: struct vm_fault containing details of the fault 1281 * 1282 * filemap_fault() is invoked via the vma operations vector for a 1283 * mapped memory region to read in file data during a page fault. 1284 * 1285 * The goto's are kind of ugly, but this streamlines the normal case of having 1286 * it in the page cache, and handles the special cases reasonably without 1287 * having a lot of duplicated code. 1288 */ 1289int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1290{ 1291 int error; 1292 struct file *file = vma->vm_file; 1293 struct address_space *mapping = file->f_mapping; 1294 struct file_ra_state *ra = &file->f_ra; 1295 struct inode *inode = mapping->host; 1296 struct page *page; 1297 unsigned long size; 1298 int did_readaround = 0; 1299 int ret = 0; 1300 1301 size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 1302 if (vmf->pgoff >= size) 1303 return VM_FAULT_SIGBUS; 1304 1305 /* If we don't want any read-ahead, don't bother */ 1306 if (VM_RandomReadHint(vma)) 1307 goto no_cached_page; 1308 1309 /* 1310 * Do we have something in the page cache already? 1311 */ 1312retry_find: 1313 page = find_lock_page(mapping, vmf->pgoff); 1314 /* 1315 * For sequential accesses, we use the generic readahead logic. 1316 */ 1317 if (VM_SequentialReadHint(vma)) { 1318 if (!page) { 1319 page_cache_sync_readahead(mapping, ra, file, 1320 vmf->pgoff, 1); 1321 page = find_lock_page(mapping, vmf->pgoff); 1322 if (!page) 1323 goto no_cached_page; 1324 } 1325 if (PageReadahead(page)) { 1326 page_cache_async_readahead(mapping, ra, file, page, 1327 vmf->pgoff, 1); 1328 } 1329 } 1330 1331 if (!page) { 1332 unsigned long ra_pages; 1333 1334 ra->mmap_miss++; 1335 1336 /* 1337 * Do we miss much more than hit in this file? If so, 1338 * stop bothering with read-ahead. It will only hurt. 1339 */ 1340 if (ra->mmap_miss > MMAP_LOTSAMISS) 1341 goto no_cached_page; 1342 1343 /* 1344 * To keep the pgmajfault counter straight, we need to 1345 * check did_readaround, as this is an inner loop. 1346 */ 1347 if (!did_readaround) { 1348 ret = VM_FAULT_MAJOR; 1349 count_vm_event(PGMAJFAULT); 1350 } 1351 did_readaround = 1; 1352 ra_pages = max_sane_readahead(file->f_ra.ra_pages); 1353 if (ra_pages) { 1354 pgoff_t start = 0; 1355 1356 if (vmf->pgoff > ra_pages / 2) 1357 start = vmf->pgoff - ra_pages / 2; 1358 do_page_cache_readahead(mapping, file, start, ra_pages); 1359 } 1360 page = find_lock_page(mapping, vmf->pgoff); 1361 if (!page) 1362 goto no_cached_page; 1363 } 1364 1365 if (!did_readaround) 1366 ra->mmap_miss--; 1367 1368 /* 1369 * We have a locked page in the page cache, now we need to check 1370 * that it's up-to-date. If not, it is going to be due to an error. 1371 */ 1372 if (unlikely(!PageUptodate(page))) 1373 goto page_not_uptodate; 1374 1375 /* Must recheck i_size under page lock */ 1376 size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 1377 if (unlikely(vmf->pgoff >= size)) { 1378 unlock_page(page); 1379 page_cache_release(page); 1380 return VM_FAULT_SIGBUS; 1381 } 1382 1383 /* 1384 * Found the page and have a reference on it. 1385 */ 1386 mark_page_accessed(page); 1387 ra->prev_pos = (loff_t)page->index << PAGE_CACHE_SHIFT; 1388 vmf->page = page; 1389 return ret | VM_FAULT_LOCKED; 1390 1391no_cached_page: 1392 /* 1393 * We're only likely to ever get here if MADV_RANDOM is in 1394 * effect. 1395 */ 1396 error = page_cache_read(file, vmf->pgoff); 1397 1398 /* 1399 * The page we want has now been added to the page cache. 1400 * In the unlikely event that someone removed it in the 1401 * meantime, we'll just come back here and read it again. 1402 */ 1403 if (error >= 0) 1404 goto retry_find; 1405 1406 /* 1407 * An error return from page_cache_read can result if the 1408 * system is low on memory, or a problem occurs while trying 1409 * to schedule I/O. 1410 */ 1411 if (error == -ENOMEM) 1412 return VM_FAULT_OOM; 1413 return VM_FAULT_SIGBUS; 1414 1415page_not_uptodate: 1416 /* IO error path */ 1417 if (!did_readaround) { 1418 ret = VM_FAULT_MAJOR; 1419 count_vm_event(PGMAJFAULT); 1420 } 1421 1422 /* 1423 * Umm, take care of errors if the page isn't up-to-date. 1424 * Try to re-read it _once_. We do this synchronously, 1425 * because there really aren't any performance issues here 1426 * and we need to check for errors. 1427 */ 1428 ClearPageError(page); 1429 error = mapping->a_ops->readpage(file, page); 1430 page_cache_release(page); 1431 1432 if (!error || error == AOP_TRUNCATED_PAGE) 1433 goto retry_find; 1434 1435 /* Things didn't work out. Return zero to tell the mm layer so. */ 1436 shrink_readahead_size_eio(file, ra); 1437 return VM_FAULT_SIGBUS; 1438} 1439EXPORT_SYMBOL(filemap_fault); 1440 1441struct vm_operations_struct generic_file_vm_ops = { 1442 .fault = filemap_fault, 1443}; 1444 1445/* This is used for a general mmap of a disk file */ 1446 1447int generic_file_mmap(struct file * file, struct vm_area_struct * vma) 1448{ 1449 struct address_space *mapping = file->f_mapping; 1450 1451 if (!mapping->a_ops->readpage) 1452 return -ENOEXEC; 1453 file_accessed(file); 1454 vma->vm_ops = &generic_file_vm_ops; 1455 vma->vm_flags |= VM_CAN_NONLINEAR; 1456 return 0; 1457} 1458 1459/* 1460 * This is for filesystems which do not implement ->writepage. 1461 */ 1462int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma) 1463{ 1464 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) 1465 return -EINVAL; 1466 return generic_file_mmap(file, vma); 1467} 1468#else 1469int generic_file_mmap(struct file * file, struct vm_area_struct * vma) 1470{ 1471 return -ENOSYS; 1472} 1473int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma) 1474{ 1475 return -ENOSYS; 1476} 1477#endif /* CONFIG_MMU */ 1478 1479EXPORT_SYMBOL(generic_file_mmap); 1480EXPORT_SYMBOL(generic_file_readonly_mmap); 1481 1482static struct page *__read_cache_page(struct address_space *mapping, 1483 pgoff_t index, 1484 int (*filler)(void *,struct page*), 1485 void *data) 1486{ 1487 struct page *page; 1488 int err; 1489repeat: 1490 page = find_get_page(mapping, index); 1491 if (!page) { 1492 page = page_cache_alloc_cold(mapping); 1493 if (!page) 1494 return ERR_PTR(-ENOMEM); 1495 err = add_to_page_cache_lru(page, mapping, index, GFP_KERNEL); 1496 if (unlikely(err)) { 1497 page_cache_release(page); 1498 if (err == -EEXIST) 1499 goto repeat; 1500 /* Presumably ENOMEM for radix tree node */ 1501 return ERR_PTR(err); 1502 } 1503 err = filler(data, page); 1504 if (err < 0) { 1505 page_cache_release(page); 1506 page = ERR_PTR(err); 1507 } 1508 } 1509 return page; 1510} 1511 1512/* 1513 * Same as read_cache_page, but don't wait for page to become unlocked 1514 * after submitting it to the filler. 1515 */ 1516struct page *read_cache_page_async(struct address_space *mapping, 1517 pgoff_t index, 1518 int (*filler)(void *,struct page*), 1519 void *data) 1520{ 1521 struct page *page; 1522 int err; 1523 1524retry: 1525 page = __read_cache_page(mapping, index, filler, data); 1526 if (IS_ERR(page)) 1527 return page; 1528 if (PageUptodate(page)) 1529 goto out; 1530 1531 lock_page(page); 1532 if (!page->mapping) { 1533 unlock_page(page); 1534 page_cache_release(page); 1535 goto retry; 1536 } 1537 if (PageUptodate(page)) { 1538 unlock_page(page); 1539 goto out; 1540 } 1541 err = filler(data, page); 1542 if (err < 0) { 1543 page_cache_release(page); 1544 return ERR_PTR(err); 1545 } 1546out: 1547 mark_page_accessed(page); 1548 return page; 1549} 1550EXPORT_SYMBOL(read_cache_page_async); 1551 1552/** 1553 * read_cache_page - read into page cache, fill it if needed 1554 * @mapping: the page's address_space 1555 * @index: the page index 1556 * @filler: function to perform the read 1557 * @data: destination for read data 1558 * 1559 * Read into the page cache. If a page already exists, and PageUptodate() is 1560 * not set, try to fill the page then wait for it to become unlocked. 1561 * 1562 * If the page does not get brought uptodate, return -EIO. 1563 */ 1564struct page *read_cache_page(struct address_space *mapping, 1565 pgoff_t index, 1566 int (*filler)(void *,struct page*), 1567 void *data) 1568{ 1569 struct page *page; 1570 1571 page = read_cache_page_async(mapping, index, filler, data); 1572 if (IS_ERR(page)) 1573 goto out; 1574 wait_on_page_locked(page); 1575 if (!PageUptodate(page)) { 1576 page_cache_release(page); 1577 page = ERR_PTR(-EIO); 1578 } 1579 out: 1580 return page; 1581} 1582EXPORT_SYMBOL(read_cache_page); 1583 1584/* 1585 * The logic we want is 1586 * 1587 * if suid or (sgid and xgrp) 1588 * remove privs 1589 */ 1590int should_remove_suid(struct dentry *dentry) 1591{ 1592 mode_t mode = dentry->d_inode->i_mode; 1593 int kill = 0; 1594 1595 /* suid always must be killed */ 1596 if (unlikely(mode & S_ISUID)) 1597 kill = ATTR_KILL_SUID; 1598 1599 /* 1600 * sgid without any exec bits is just a mandatory locking mark; leave 1601 * it alone. If some exec bits are set, it's a real sgid; kill it. 1602 */ 1603 if (unlikely((mode & S_ISGID) && (mode & S_IXGRP))) 1604 kill |= ATTR_KILL_SGID; 1605 1606 if (unlikely(kill && !capable(CAP_FSETID))) 1607 return kill; 1608 1609 return 0; 1610} 1611EXPORT_SYMBOL(should_remove_suid); 1612 1613int __remove_suid(struct dentry *dentry, int kill) 1614{ 1615 struct iattr newattrs; 1616 1617 newattrs.ia_valid = ATTR_FORCE | kill; 1618 return notify_change(dentry, &newattrs); 1619} 1620 1621int remove_suid(struct dentry *dentry) 1622{ 1623 int killsuid = should_remove_suid(dentry); 1624 int killpriv = security_inode_need_killpriv(dentry); 1625 int error = 0; 1626 1627 if (killpriv < 0) 1628 return killpriv; 1629 if (killpriv) 1630 error = security_inode_killpriv(dentry); 1631 if (!error && killsuid) 1632 error = __remove_suid(dentry, killsuid); 1633 1634 return error; 1635} 1636EXPORT_SYMBOL(remove_suid); 1637 1638static size_t __iovec_copy_from_user_inatomic(char *vaddr, 1639 const struct iovec *iov, size_t base, size_t bytes) 1640{ 1641 size_t copied = 0, left = 0; 1642 1643 while (bytes) { 1644 char __user *buf = iov->iov_base + base; 1645 int copy = min(bytes, iov->iov_len - base); 1646 1647 base = 0; 1648 left = __copy_from_user_inatomic_nocache(vaddr, buf, copy); 1649 copied += copy; 1650 bytes -= copy; 1651 vaddr += copy; 1652 iov++; 1653 1654 if (unlikely(left)) 1655 break; 1656 } 1657 return copied - left; 1658} 1659 1660/* 1661 * Copy as much as we can into the page and return the number of bytes which 1662 * were sucessfully copied. If a fault is encountered then return the number of 1663 * bytes which were copied. 1664 */ 1665size_t iov_iter_copy_from_user_atomic(struct page *page, 1666 struct iov_iter *i, unsigned long offset, size_t bytes) 1667{ 1668 char *kaddr; 1669 size_t copied; 1670 1671 BUG_ON(!in_atomic()); 1672 kaddr = kmap_atomic(page, KM_USER0); 1673 if (likely(i->nr_segs == 1)) { 1674 int left; 1675 char __user *buf = i->iov->iov_base + i->iov_offset; 1676 left = __copy_from_user_inatomic_nocache(kaddr + offset, 1677 buf, bytes); 1678 copied = bytes - left; 1679 } else { 1680 copied = __iovec_copy_from_user_inatomic(kaddr + offset, 1681 i->iov, i->iov_offset, bytes); 1682 } 1683 kunmap_atomic(kaddr, KM_USER0); 1684 1685 return copied; 1686} 1687EXPORT_SYMBOL(iov_iter_copy_from_user_atomic); 1688 1689/* 1690 * This has the same sideeffects and return value as 1691 * iov_iter_copy_from_user_atomic(). 1692 * The difference is that it attempts to resolve faults. 1693 * Page must not be locked. 1694 */ 1695size_t iov_iter_copy_from_user(struct page *page, 1696 struct iov_iter *i, unsigned long offset, size_t bytes) 1697{ 1698 char *kaddr; 1699 size_t copied; 1700 1701 kaddr = kmap(page); 1702 if (likely(i->nr_segs == 1)) { 1703 int left; 1704 char __user *buf = i->iov->iov_base + i->iov_offset; 1705 left = __copy_from_user_nocache(kaddr + offset, buf, bytes); 1706 copied = bytes - left; 1707 } else { 1708 copied = __iovec_copy_from_user_inatomic(kaddr + offset, 1709 i->iov, i->iov_offset, bytes); 1710 } 1711 kunmap(page); 1712 return copied; 1713} 1714EXPORT_SYMBOL(iov_iter_copy_from_user); 1715 1716static void __iov_iter_advance_iov(struct iov_iter *i, size_t bytes) 1717{ 1718 if (likely(i->nr_segs == 1)) { 1719 i->iov_offset += bytes; 1720 } else { 1721 const struct iovec *iov = i->iov; 1722 size_t base = i->iov_offset; 1723 1724 while (bytes) { 1725 int copy = min(bytes, iov->iov_len - base); 1726 1727 bytes -= copy; 1728 base += copy; 1729 if (iov->iov_len == base) { 1730 iov++; 1731 base = 0; 1732 } 1733 } 1734 i->iov = iov; 1735 i->iov_offset = base; 1736 } 1737} 1738 1739void iov_iter_advance(struct iov_iter *i, size_t bytes) 1740{ 1741 BUG_ON(i->count < bytes); 1742 1743 __iov_iter_advance_iov(i, bytes); 1744 i->count -= bytes; 1745} 1746EXPORT_SYMBOL(iov_iter_advance); 1747 1748/* 1749 * Fault in the first iovec of the given iov_iter, to a maximum length 1750 * of bytes. Returns 0 on success, or non-zero if the memory could not be 1751 * accessed (ie. because it is an invalid address). 1752 * 1753 * writev-intensive code may want this to prefault several iovecs -- that 1754 * would be possible (callers must not rely on the fact that _only_ the 1755 * first iovec will be faulted with the current implementation). 1756 */ 1757int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes) 1758{ 1759 char __user *buf = i->iov->iov_base + i->iov_offset; 1760 bytes = min(bytes, i->iov->iov_len - i->iov_offset); 1761 return fault_in_pages_readable(buf, bytes); 1762} 1763EXPORT_SYMBOL(iov_iter_fault_in_readable); 1764 1765/* 1766 * Return the count of just the current iov_iter segment. 1767 */ 1768size_t iov_iter_single_seg_count(struct iov_iter *i) 1769{ 1770 const struct iovec *iov = i->iov; 1771 if (i->nr_segs == 1) 1772 return i->count; 1773 else 1774 return min(i->count, iov->iov_len - i->iov_offset); 1775} 1776EXPORT_SYMBOL(iov_iter_single_seg_count); 1777 1778/* 1779 * Performs necessary checks before doing a write 1780 * 1781 * Can adjust writing position or amount of bytes to write. 1782 * Returns appropriate error code that caller should return or 1783 * zero in case that write should be allowed. 1784 */ 1785inline int generic_write_checks(struct file *file, loff_t *pos, size_t *count, int isblk) 1786{ 1787 struct inode *inode = file->f_mapping->host; 1788 unsigned long limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur; 1789 1790 if (unlikely(*pos < 0)) 1791 return -EINVAL; 1792 1793 if (!isblk) { 1794 /* FIXME: this is for backwards compatibility with 2.4 */ 1795 if (file->f_flags & O_APPEND) 1796 *pos = i_size_read(inode); 1797 1798 if (limit != RLIM_INFINITY) { 1799 if (*pos >= limit) { 1800 send_sig(SIGXFSZ, current, 0); 1801 return -EFBIG; 1802 } 1803 if (*count > limit - (typeof(limit))*pos) { 1804 *count = limit - (typeof(limit))*pos; 1805 } 1806 } 1807 } 1808 1809 /* 1810 * LFS rule 1811 */ 1812 if (unlikely(*pos + *count > MAX_NON_LFS && 1813 !(file->f_flags & O_LARGEFILE))) { 1814 if (*pos >= MAX_NON_LFS) { 1815 return -EFBIG; 1816 } 1817 if (*count > MAX_NON_LFS - (unsigned long)*pos) { 1818 *count = MAX_NON_LFS - (unsigned long)*pos; 1819 } 1820 } 1821 1822 /* 1823 * Are we about to exceed the fs block limit ? 1824 * 1825 * If we have written data it becomes a short write. If we have 1826 * exceeded without writing data we send a signal and return EFBIG. 1827 * Linus frestrict idea will clean these up nicely.. 1828 */ 1829 if (likely(!isblk)) { 1830 if (unlikely(*pos >= inode->i_sb->s_maxbytes)) { 1831 if (*count || *pos > inode->i_sb->s_maxbytes) { 1832 return -EFBIG; 1833 } 1834 /* zero-length writes at ->s_maxbytes are OK */ 1835 } 1836 1837 if (unlikely(*pos + *count > inode->i_sb->s_maxbytes)) 1838 *count = inode->i_sb->s_maxbytes - *pos; 1839 } else { 1840#ifdef CONFIG_BLOCK 1841 loff_t isize; 1842 if (bdev_read_only(I_BDEV(inode))) 1843 return -EPERM; 1844 isize = i_size_read(inode); 1845 if (*pos >= isize) { 1846 if (*count || *pos > isize) 1847 return -ENOSPC; 1848 } 1849 1850 if (*pos + *count > isize) 1851 *count = isize - *pos; 1852#else 1853 return -EPERM; 1854#endif 1855 } 1856 return 0; 1857} 1858EXPORT_SYMBOL(generic_write_checks); 1859 1860int pagecache_write_begin(struct file *file, struct address_space *mapping, 1861 loff_t pos, unsigned len, unsigned flags, 1862 struct page **pagep, void **fsdata) 1863{ 1864 const struct address_space_operations *aops = mapping->a_ops; 1865 1866 if (aops->write_begin) { 1867 return aops->write_begin(file, mapping, pos, len, flags, 1868 pagep, fsdata); 1869 } else { 1870 int ret; 1871 pgoff_t index = pos >> PAGE_CACHE_SHIFT; 1872 unsigned offset = pos & (PAGE_CACHE_SIZE - 1); 1873 struct inode *inode = mapping->host; 1874 struct page *page; 1875again: 1876 page = __grab_cache_page(mapping, index); 1877 *pagep = page; 1878 if (!page) 1879 return -ENOMEM; 1880 1881 if (flags & AOP_FLAG_UNINTERRUPTIBLE && !PageUptodate(page)) { 1882 /* 1883 * There is no way to resolve a short write situation 1884 * for a !Uptodate page (except by double copying in 1885 * the caller done by generic_perform_write_2copy). 1886 * 1887 * Instead, we have to bring it uptodate here. 1888 */ 1889 ret = aops->readpage(file, page); 1890 page_cache_release(page); 1891 if (ret) { 1892 if (ret == AOP_TRUNCATED_PAGE) 1893 goto again; 1894 return ret; 1895 } 1896 goto again; 1897 } 1898 1899 ret = aops->prepare_write(file, page, offset, offset+len); 1900 if (ret) { 1901 unlock_page(page); 1902 page_cache_release(page); 1903 if (pos + len > inode->i_size) 1904 vmtruncate(inode, inode->i_size); 1905 } 1906 return ret; 1907 } 1908} 1909EXPORT_SYMBOL(pagecache_write_begin); 1910 1911int pagecache_write_end(struct file *file, struct address_space *mapping, 1912 loff_t pos, unsigned len, unsigned copied, 1913 struct page *page, void *fsdata) 1914{ 1915 const struct address_space_operations *aops = mapping->a_ops; 1916 int ret; 1917 1918 if (aops->write_end) { 1919 mark_page_accessed(page); 1920 ret = aops->write_end(file, mapping, pos, len, copied, 1921 page, fsdata); 1922 } else { 1923 unsigned offset = pos & (PAGE_CACHE_SIZE - 1); 1924 struct inode *inode = mapping->host; 1925 1926 flush_dcache_page(page); 1927 ret = aops->commit_write(file, page, offset, offset+len); 1928 unlock_page(page); 1929 mark_page_accessed(page); 1930 page_cache_release(page); 1931 1932 if (ret < 0) { 1933 if (pos + len > inode->i_size) 1934 vmtruncate(inode, inode->i_size); 1935 } else if (ret > 0) 1936 ret = min_t(size_t, copied, ret); 1937 else 1938 ret = copied; 1939 } 1940 1941 return ret; 1942} 1943EXPORT_SYMBOL(pagecache_write_end); 1944 1945ssize_t 1946generic_file_direct_write(struct kiocb *iocb, const struct iovec *iov, 1947 unsigned long *nr_segs, loff_t pos, loff_t *ppos, 1948 size_t count, size_t ocount) 1949{ 1950 struct file *file = iocb->ki_filp; 1951 struct address_space *mapping = file->f_mapping; 1952 struct inode *inode = mapping->host; 1953 ssize_t written; 1954 1955 if (count != ocount) 1956 *nr_segs = iov_shorten((struct iovec *)iov, *nr_segs, count); 1957 1958 written = generic_file_direct_IO(WRITE, iocb, iov, pos, *nr_segs); 1959 if (written > 0) { 1960 loff_t end = pos + written; 1961 if (end > i_size_read(inode) && !S_ISBLK(inode->i_mode)) { 1962 i_size_write(inode, end); 1963 mark_inode_dirty(inode); 1964 } 1965 *ppos = end; 1966 } 1967 1968 /* 1969 * Sync the fs metadata but not the minor inode changes and 1970 * of course not the data as we did direct DMA for the IO. 1971 * i_mutex is held, which protects generic_osync_inode() from 1972 * livelocking. AIO O_DIRECT ops attempt to sync metadata here. 1973 */ 1974 if ((written >= 0 || written == -EIOCBQUEUED) && 1975 ((file->f_flags & O_SYNC) || IS_SYNC(inode))) { 1976 int err = generic_osync_inode(inode, mapping, OSYNC_METADATA); 1977 if (err < 0) 1978 written = err; 1979 } 1980 return written; 1981} 1982EXPORT_SYMBOL(generic_file_direct_write); 1983 1984/* 1985 * Find or create a page at the given pagecache position. Return the locked 1986 * page. This function is specifically for buffered writes. 1987 */ 1988struct page *__grab_cache_page(struct address_space *mapping, pgoff_t index) 1989{ 1990 int status; 1991 struct page *page; 1992repeat: 1993 page = find_lock_page(mapping, index); 1994 if (likely(page)) 1995 return page; 1996 1997 page = page_cache_alloc(mapping); 1998 if (!page) 1999 return NULL; 2000 status = add_to_page_cache_lru(page, mapping, index, GFP_KERNEL); 2001 if (unlikely(status)) { 2002 page_cache_release(page); 2003 if (status == -EEXIST) 2004 goto repeat; 2005 return NULL; 2006 } 2007 return page; 2008} 2009EXPORT_SYMBOL(__grab_cache_page); 2010 2011static ssize_t generic_perform_write_2copy(struct file *file, 2012 struct iov_iter *i, loff_t pos) 2013{ 2014 struct address_space *mapping = file->f_mapping; 2015 const struct address_space_operations *a_ops = mapping->a_ops; 2016 struct inode *inode = mapping->host; 2017 long status = 0; 2018 ssize_t written = 0; 2019 2020 do { 2021 struct page *src_page; 2022 struct page *page; 2023 pgoff_t index; /* Pagecache index for current page */ 2024 unsigned long offset; /* Offset into pagecache page */ 2025 unsigned long bytes; /* Bytes to write to page */ 2026 size_t copied; /* Bytes copied from user */ 2027 2028 offset = (pos & (PAGE_CACHE_SIZE - 1)); 2029 index = pos >> PAGE_CACHE_SHIFT; 2030 bytes = min_t(unsigned long, PAGE_CACHE_SIZE - offset, 2031 iov_iter_count(i)); 2032 2033 /* 2034 * a non-NULL src_page indicates that we're doing the 2035 * copy via get_user_pages and kmap. 2036 */ 2037 src_page = NULL; 2038 2039 /* 2040 * Bring in the user page that we will copy from _first_. 2041 * Otherwise there's a nasty deadlock on copying from the 2042 * same page as we're writing to, without it being marked 2043 * up-to-date. 2044 * 2045 * Not only is this an optimisation, but it is also required 2046 * to check that the address is actually valid, when atomic 2047 * usercopies are used, below. 2048 */ 2049 if (unlikely(iov_iter_fault_in_readable(i, bytes))) { 2050 status = -EFAULT; 2051 break; 2052 } 2053 2054 page = __grab_cache_page(mapping, index); 2055 if (!page) { 2056 status = -ENOMEM; 2057 break; 2058 } 2059 2060 /* 2061 * non-uptodate pages cannot cope with short copies, and we 2062 * cannot take a pagefault with the destination page locked. 2063 * So pin the source page to copy it. 2064 */ 2065 if (!PageUptodate(page) && !segment_eq(get_fs(), KERNEL_DS)) { 2066 unlock_page(page); 2067 2068 src_page = alloc_page(GFP_KERNEL); 2069 if (!src_page) { 2070 page_cache_release(page); 2071 status = -ENOMEM; 2072 break; 2073 } 2074 2075 /* 2076 * Cannot get_user_pages with a page locked for the 2077 * same reason as we can't take a page fault with a 2078 * page locked (as explained below). 2079 */ 2080 copied = iov_iter_copy_from_user(src_page, i, 2081 offset, bytes); 2082 if (unlikely(copied == 0)) { 2083 status = -EFAULT; 2084 page_cache_release(page); 2085 page_cache_release(src_page); 2086 break; 2087 } 2088 bytes = copied; 2089 2090 lock_page(page); 2091 /* 2092 * Can't handle the page going uptodate here, because 2093 * that means we would use non-atomic usercopies, which 2094 * zero out the tail of the page, which can cause 2095 * zeroes to become transiently visible. We could just 2096 * use a non-zeroing copy, but the APIs aren't too 2097 * consistent. 2098 */ 2099 if (unlikely(!page->mapping || PageUptodate(page))) { 2100 unlock_page(page); 2101 page_cache_release(page); 2102 page_cache_release(src_page); 2103 continue; 2104 } 2105 } 2106 2107 status = a_ops->prepare_write(file, page, offset, offset+bytes); 2108 if (unlikely(status)) 2109 goto fs_write_aop_error; 2110 2111 if (!src_page) { 2112 /* 2113 * Must not enter the pagefault handler here, because 2114 * we hold the page lock, so we might recursively 2115 * deadlock on the same lock, or get an ABBA deadlock 2116 * against a different lock, or against the mmap_sem 2117 * (which nests outside the page lock). So increment 2118 * preempt count, and use _atomic usercopies. 2119 * 2120 * The page is uptodate so we are OK to encounter a 2121 * short copy: if unmodified parts of the page are 2122 * marked dirty and written out to disk, it doesn't 2123 * really matter. 2124 */ 2125 pagefault_disable(); 2126 copied = iov_iter_copy_from_user_atomic(page, i, 2127 offset, bytes); 2128 pagefault_enable(); 2129 } else { 2130 void *src, *dst; 2131 src = kmap_atomic(src_page, KM_USER0); 2132 dst = kmap_atomic(page, KM_USER1); 2133 memcpy(dst + offset, src + offset, bytes); 2134 kunmap_atomic(dst, KM_USER1); 2135 kunmap_atomic(src, KM_USER0); 2136 copied = bytes; 2137 } 2138 flush_dcache_page(page); 2139 2140 status = a_ops->commit_write(file, page, offset, offset+bytes); 2141 if (unlikely(status < 0)) 2142 goto fs_write_aop_error; 2143 if (unlikely(status > 0)) /* filesystem did partial write */ 2144 copied = min_t(size_t, copied, status); 2145 2146 unlock_page(page); 2147 mark_page_accessed(page); 2148 page_cache_release(page); 2149 if (src_page) 2150 page_cache_release(src_page); 2151 2152 iov_iter_advance(i, copied); 2153 pos += copied; 2154 written += copied; 2155 2156 balance_dirty_pages_ratelimited(mapping); 2157 cond_resched(); 2158 continue; 2159 2160fs_write_aop_error: 2161 unlock_page(page); 2162 page_cache_release(page); 2163 if (src_page) 2164 page_cache_release(src_page); 2165 2166 /* 2167 * prepare_write() may have instantiated a few blocks 2168 * outside i_size. Trim these off again. Don't need 2169 * i_size_read because we hold i_mutex. 2170 */ 2171 if (pos + bytes > inode->i_size) 2172 vmtruncate(inode, inode->i_size); 2173 break; 2174 } while (iov_iter_count(i)); 2175 2176 return written ? written : status; 2177} 2178 2179static ssize_t generic_perform_write(struct file *file, 2180 struct iov_iter *i, loff_t pos) 2181{ 2182 struct address_space *mapping = file->f_mapping; 2183 const struct address_space_operations *a_ops = mapping->a_ops; 2184 long status = 0; 2185 ssize_t written = 0; 2186 unsigned int flags = 0; 2187 2188 /* 2189 * Copies from kernel address space cannot fail (NFSD is a big user). 2190 */ 2191 if (segment_eq(get_fs(), KERNEL_DS)) 2192 flags |= AOP_FLAG_UNINTERRUPTIBLE; 2193 2194 do { 2195 struct page *page; 2196 pgoff_t index; /* Pagecache index for current page */ 2197 unsigned long offset; /* Offset into pagecache page */ 2198 unsigned long bytes; /* Bytes to write to page */ 2199 size_t copied; /* Bytes copied from user */ 2200 void *fsdata; 2201 2202 offset = (pos & (PAGE_CACHE_SIZE - 1)); 2203 index = pos >> PAGE_CACHE_SHIFT; 2204 bytes = min_t(unsigned long, PAGE_CACHE_SIZE - offset, 2205 iov_iter_count(i)); 2206 2207again: 2208 2209 /* 2210 * Bring in the user page that we will copy from _first_. 2211 * Otherwise there's a nasty deadlock on copying from the 2212 * same page as we're writing to, without it being marked 2213 * up-to-date. 2214 * 2215 * Not only is this an optimisation, but it is also required 2216 * to check that the address is actually valid, when atomic 2217 * usercopies are used, below. 2218 */ 2219 if (unlikely(iov_iter_fault_in_readable(i, bytes))) { 2220 status = -EFAULT; 2221 break; 2222 } 2223 2224 status = a_ops->write_begin(file, mapping, pos, bytes, flags, 2225 &page, &fsdata); 2226 if (unlikely(status)) 2227 break; 2228 2229 pagefault_disable(); 2230 copied = iov_iter_copy_from_user_atomic(page, i, offset, bytes); 2231 pagefault_enable(); 2232 flush_dcache_page(page); 2233 2234 status = a_ops->write_end(file, mapping, pos, bytes, copied, 2235 page, fsdata); 2236 if (unlikely(status < 0)) 2237 break; 2238 copied = status; 2239 2240 cond_resched(); 2241 2242 if (unlikely(copied == 0)) { 2243 /* 2244 * If we were unable to copy any data at all, we must 2245 * fall back to a single segment length write. 2246 * 2247 * If we didn't fallback here, we could livelock 2248 * because not all segments in the iov can be copied at 2249 * once without a pagefault. 2250 */ 2251 bytes = min_t(unsigned long, PAGE_CACHE_SIZE - offset, 2252 iov_iter_single_seg_count(i)); 2253 goto again; 2254 } 2255 iov_iter_advance(i, copied); 2256 pos += copied; 2257 written += copied; 2258 2259 balance_dirty_pages_ratelimited(mapping); 2260 2261 } while (iov_iter_count(i)); 2262 2263 return written ? written : status; 2264} 2265 2266ssize_t 2267generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, 2268 unsigned long nr_segs, loff_t pos, loff_t *ppos, 2269 size_t count, ssize_t written) 2270{ 2271 struct file *file = iocb->ki_filp; 2272 struct address_space *mapping = file->f_mapping; 2273 const struct address_space_operations *a_ops = mapping->a_ops; 2274 struct inode *inode = mapping->host; 2275 ssize_t status; 2276 struct iov_iter i; 2277 2278 iov_iter_init(&i, iov, nr_segs, count, written); 2279 if (a_ops->write_begin) 2280 status = generic_perform_write(file, &i, pos); 2281 else 2282 status = generic_perform_write_2copy(file, &i, pos); 2283 2284 if (likely(status >= 0)) { 2285 written += status; 2286 *ppos = pos + status; 2287 2288 /* 2289 * For now, when the user asks for O_SYNC, we'll actually give 2290 * O_DSYNC 2291 */ 2292 if (unlikely((file->f_flags & O_SYNC) || IS_SYNC(inode))) { 2293 if (!a_ops->writepage || !is_sync_kiocb(iocb)) 2294 status = generic_osync_inode(inode, mapping, 2295 OSYNC_METADATA|OSYNC_DATA); 2296 } 2297 } 2298 2299 /* 2300 * If we get here for O_DIRECT writes then we must have fallen through 2301 * to buffered writes (block instantiation inside i_size). So we sync 2302 * the file data here, to try to honour O_DIRECT expectations. 2303 */ 2304 if (unlikely(file->f_flags & O_DIRECT) && written) 2305 status = filemap_write_and_wait(mapping); 2306 2307 return written ? written : status; 2308} 2309EXPORT_SYMBOL(generic_file_buffered_write); 2310 2311static ssize_t 2312__generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov, 2313 unsigned long nr_segs, loff_t *ppos) 2314{ 2315 struct file *file = iocb->ki_filp; 2316 struct address_space * mapping = file->f_mapping; 2317 size_t ocount; /* original count */ 2318 size_t count; /* after file limit checks */ 2319 struct inode *inode = mapping->host; 2320 loff_t pos; 2321 ssize_t written; 2322 ssize_t err; 2323 2324 ocount = 0; 2325 err = generic_segment_checks(iov, &nr_segs, &ocount, VERIFY_READ); 2326 if (err) 2327 return err; 2328 2329 count = ocount; 2330 pos = *ppos; 2331 2332 vfs_check_frozen(inode->i_sb, SB_FREEZE_WRITE); 2333 2334 /* We can write back this queue in page reclaim */ 2335 current->backing_dev_info = mapping->backing_dev_info; 2336 written = 0; 2337 2338 err = generic_write_checks(file, &pos, &count, S_ISBLK(inode->i_mode)); 2339 if (err) 2340 goto out; 2341 2342 if (count == 0) 2343 goto out; 2344 2345 err = remove_suid(file->f_path.dentry); 2346 if (err) 2347 goto out; 2348 2349 file_update_time(file); 2350 2351 /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */ 2352 if (unlikely(file->f_flags & O_DIRECT)) { 2353 loff_t endbyte; 2354 ssize_t written_buffered; 2355 2356 written = generic_file_direct_write(iocb, iov, &nr_segs, pos, 2357 ppos, count, ocount); 2358 if (written < 0 || written == count) 2359 goto out; 2360 /* 2361 * direct-io write to a hole: fall through to buffered I/O 2362 * for completing the rest of the request. 2363 */ 2364 pos += written; 2365 count -= written; 2366 written_buffered = generic_file_buffered_write(iocb, iov, 2367 nr_segs, pos, ppos, count, 2368 written); 2369 /* 2370 * If generic_file_buffered_write() retuned a synchronous error 2371 * then we want to return the number of bytes which were 2372 * direct-written, or the error code if that was zero. Note 2373 * that this differs from normal direct-io semantics, which 2374 * will return -EFOO even if some bytes were written. 2375 */ 2376 if (written_buffered < 0) { 2377 err = written_buffered; 2378 goto out; 2379 } 2380 2381 /* 2382 * We need to ensure that the page cache pages are written to 2383 * disk and invalidated to preserve the expected O_DIRECT 2384 * semantics. 2385 */ 2386 endbyte = pos + written_buffered - written - 1; 2387 err = do_sync_mapping_range(file->f_mapping, pos, endbyte, 2388 SYNC_FILE_RANGE_WAIT_BEFORE| 2389 SYNC_FILE_RANGE_WRITE| 2390 SYNC_FILE_RANGE_WAIT_AFTER); 2391 if (err == 0) { 2392 written = written_buffered; 2393 invalidate_mapping_pages(mapping, 2394 pos >> PAGE_CACHE_SHIFT, 2395 endbyte >> PAGE_CACHE_SHIFT); 2396 } else { 2397 /* 2398 * We don't know how much we wrote, so just return 2399 * the number of bytes which were direct-written 2400 */ 2401 } 2402 } else { 2403 written = generic_file_buffered_write(iocb, iov, nr_segs, 2404 pos, ppos, count, written); 2405 } 2406out: 2407 current->backing_dev_info = NULL; 2408 return written ? written : err; 2409} 2410 2411ssize_t generic_file_aio_write_nolock(struct kiocb *iocb, 2412 const struct iovec *iov, unsigned long nr_segs, loff_t pos) 2413{ 2414 struct file *file = iocb->ki_filp; 2415 struct address_space *mapping = file->f_mapping; 2416 struct inode *inode = mapping->host; 2417 ssize_t ret; 2418 2419 BUG_ON(iocb->ki_pos != pos); 2420 2421 ret = __generic_file_aio_write_nolock(iocb, iov, nr_segs, 2422 &iocb->ki_pos); 2423 2424 if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) { 2425 ssize_t err; 2426 2427 err = sync_page_range_nolock(inode, mapping, pos, ret); 2428 if (err < 0) 2429 ret = err; 2430 } 2431 return ret; 2432} 2433EXPORT_SYMBOL(generic_file_aio_write_nolock); 2434 2435ssize_t generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov, 2436 unsigned long nr_segs, loff_t pos) 2437{ 2438 struct file *file = iocb->ki_filp; 2439 struct address_space *mapping = file->f_mapping; 2440 struct inode *inode = mapping->host; 2441 ssize_t ret; 2442 2443 BUG_ON(iocb->ki_pos != pos); 2444 2445 mutex_lock(&inode->i_mutex); 2446 ret = __generic_file_aio_write_nolock(iocb, iov, nr_segs, 2447 &iocb->ki_pos); 2448 mutex_unlock(&inode->i_mutex); 2449 2450 if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) { 2451 ssize_t err; 2452 2453 err = sync_page_range(inode, mapping, pos, ret); 2454 if (err < 0) 2455 ret = err; 2456 } 2457 return ret; 2458} 2459EXPORT_SYMBOL(generic_file_aio_write); 2460 2461/* 2462 * Called under i_mutex for writes to S_ISREG files. Returns -EIO if something 2463 * went wrong during pagecache shootdown. 2464 */ 2465static ssize_t 2466generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, 2467 loff_t offset, unsigned long nr_segs) 2468{ 2469 struct file *file = iocb->ki_filp; 2470 struct address_space *mapping = file->f_mapping; 2471 ssize_t retval; 2472 size_t write_len; 2473 pgoff_t end = 0; /* silence gcc */ 2474 2475 /* 2476 * If it's a write, unmap all mmappings of the file up-front. This 2477 * will cause any pte dirty bits to be propagated into the pageframes 2478 * for the subsequent filemap_write_and_wait(). 2479 */ 2480 if (rw == WRITE) { 2481 write_len = iov_length(iov, nr_segs); 2482 end = (offset + write_len - 1) >> PAGE_CACHE_SHIFT; 2483 if (mapping_mapped(mapping)) 2484 unmap_mapping_range(mapping, offset, write_len, 0); 2485 } 2486 2487 retval = filemap_write_and_wait(mapping); 2488 if (retval) 2489 goto out; 2490 2491 /* 2492 * After a write we want buffered reads to be sure to go to disk to get 2493 * the new data. We invalidate clean cached page from the region we're 2494 * about to write. We do this *before* the write so that we can return 2495 * -EIO without clobbering -EIOCBQUEUED from ->direct_IO(). 2496 */ 2497 if (rw == WRITE && mapping->nrpages) { 2498 retval = invalidate_inode_pages2_range(mapping, 2499 offset >> PAGE_CACHE_SHIFT, end); 2500 if (retval) 2501 goto out; 2502 } 2503 2504 retval = mapping->a_ops->direct_IO(rw, iocb, iov, offset, nr_segs); 2505 2506 /* 2507 * Finally, try again to invalidate clean pages which might have been 2508 * cached by non-direct readahead, or faulted in by get_user_pages() 2509 * if the source of the write was an mmap'ed region of the file 2510 * we're writing. Either one is a pretty crazy thing to do, 2511 * so we don't support it 100%. If this invalidation 2512 * fails, tough, the write still worked... 2513 */ 2514 if (rw == WRITE && mapping->nrpages) { 2515 invalidate_inode_pages2_range(mapping, offset >> PAGE_CACHE_SHIFT, end); 2516 } 2517out: 2518 return retval; 2519} 2520 2521/** 2522 * try_to_release_page() - release old fs-specific metadata on a page 2523 * 2524 * @page: the page which the kernel is trying to free 2525 * @gfp_mask: memory allocation flags (and I/O mode) 2526 * 2527 * The address_space is to try to release any data against the page 2528 * (presumably at page->private). If the release was successful, return `1'. 2529 * Otherwise return zero. 2530 * 2531 * The @gfp_mask argument specifies whether I/O may be performed to release 2532 * this page (__GFP_IO), and whether the call may block (__GFP_WAIT). 2533 * 2534 * NOTE: @gfp_mask may go away, and this function may become non-blocking. 2535 */ 2536int try_to_release_page(struct page *page, gfp_t gfp_mask) 2537{ 2538 struct address_space * const mapping = page->mapping; 2539 2540 BUG_ON(!PageLocked(page)); 2541 if (PageWriteback(page)) 2542 return 0; 2543 2544 if (mapping && mapping->a_ops->releasepage) 2545 return mapping->a_ops->releasepage(page, gfp_mask); 2546 return try_to_free_buffers(page); 2547} 2548 2549EXPORT_SYMBOL(try_to_release_page); 2550