filemap.c revision b0cfbd995d091b10841eeb948976f5d1fbf13cdd
1/* 2 * linux/mm/filemap.c 3 * 4 * Copyright (C) 1994-1999 Linus Torvalds 5 */ 6 7/* 8 * This file handles the generic file mmap semantics used by 9 * most "normal" filesystems (but you don't /have/ to use this: 10 * the NFS filesystem used to do this differently, for example) 11 */ 12#include <linux/config.h> 13#include <linux/module.h> 14#include <linux/slab.h> 15#include <linux/compiler.h> 16#include <linux/fs.h> 17#include <linux/aio.h> 18#include <linux/kernel_stat.h> 19#include <linux/mm.h> 20#include <linux/swap.h> 21#include <linux/mman.h> 22#include <linux/pagemap.h> 23#include <linux/file.h> 24#include <linux/uio.h> 25#include <linux/hash.h> 26#include <linux/writeback.h> 27#include <linux/pagevec.h> 28#include <linux/blkdev.h> 29#include <linux/security.h> 30#include <linux/syscalls.h> 31#include "filemap.h" 32/* 33 * FIXME: remove all knowledge of the buffer layer from the core VM 34 */ 35#include <linux/buffer_head.h> /* for generic_osync_inode */ 36 37#include <asm/uaccess.h> 38#include <asm/mman.h> 39 40/* 41 * Shared mappings implemented 30.11.1994. It's not fully working yet, 42 * though. 43 * 44 * Shared mappings now work. 15.8.1995 Bruno. 45 * 46 * finished 'unifying' the page and buffer cache and SMP-threaded the 47 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com> 48 * 49 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de> 50 */ 51 52/* 53 * Lock ordering: 54 * 55 * ->i_mmap_lock (vmtruncate) 56 * ->private_lock (__free_pte->__set_page_dirty_buffers) 57 * ->swap_list_lock 58 * ->swap_device_lock (exclusive_swap_page, others) 59 * ->mapping->tree_lock 60 * 61 * ->i_sem 62 * ->i_mmap_lock (truncate->unmap_mapping_range) 63 * 64 * ->mmap_sem 65 * ->i_mmap_lock 66 * ->page_table_lock (various places, mainly in mmap.c) 67 * ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock) 68 * 69 * ->mmap_sem 70 * ->lock_page (access_process_vm) 71 * 72 * ->mmap_sem 73 * ->i_sem (msync) 74 * 75 * ->i_sem 76 * ->i_alloc_sem (various) 77 * 78 * ->inode_lock 79 * ->sb_lock (fs/fs-writeback.c) 80 * ->mapping->tree_lock (__sync_single_inode) 81 * 82 * ->i_mmap_lock 83 * ->anon_vma.lock (vma_adjust) 84 * 85 * ->anon_vma.lock 86 * ->page_table_lock (anon_vma_prepare and various) 87 * 88 * ->page_table_lock 89 * ->swap_device_lock (try_to_unmap_one) 90 * ->private_lock (try_to_unmap_one) 91 * ->tree_lock (try_to_unmap_one) 92 * ->zone.lru_lock (follow_page->mark_page_accessed) 93 * ->private_lock (page_remove_rmap->set_page_dirty) 94 * ->tree_lock (page_remove_rmap->set_page_dirty) 95 * ->inode_lock (page_remove_rmap->set_page_dirty) 96 * ->inode_lock (zap_pte_range->set_page_dirty) 97 * ->private_lock (zap_pte_range->__set_page_dirty_buffers) 98 * 99 * ->task->proc_lock 100 * ->dcache_lock (proc_pid_lookup) 101 */ 102 103/* 104 * Remove a page from the page cache and free it. Caller has to make 105 * sure the page is locked and that nobody else uses it - or that usage 106 * is safe. The caller must hold a write_lock on the mapping's tree_lock. 107 */ 108void __remove_from_page_cache(struct page *page) 109{ 110 struct address_space *mapping = page->mapping; 111 112 radix_tree_delete(&mapping->page_tree, page->index); 113 page->mapping = NULL; 114 mapping->nrpages--; 115 pagecache_acct(-1); 116} 117 118void remove_from_page_cache(struct page *page) 119{ 120 struct address_space *mapping = page->mapping; 121 122 BUG_ON(!PageLocked(page)); 123 124 write_lock_irq(&mapping->tree_lock); 125 __remove_from_page_cache(page); 126 write_unlock_irq(&mapping->tree_lock); 127} 128 129static int sync_page(void *word) 130{ 131 struct address_space *mapping; 132 struct page *page; 133 134 page = container_of((page_flags_t *)word, struct page, flags); 135 136 /* 137 * page_mapping() is being called without PG_locked held. 138 * Some knowledge of the state and use of the page is used to 139 * reduce the requirements down to a memory barrier. 140 * The danger here is of a stale page_mapping() return value 141 * indicating a struct address_space different from the one it's 142 * associated with when it is associated with one. 143 * After smp_mb(), it's either the correct page_mapping() for 144 * the page, or an old page_mapping() and the page's own 145 * page_mapping() has gone NULL. 146 * The ->sync_page() address_space operation must tolerate 147 * page_mapping() going NULL. By an amazing coincidence, 148 * this comes about because none of the users of the page 149 * in the ->sync_page() methods make essential use of the 150 * page_mapping(), merely passing the page down to the backing 151 * device's unplug functions when it's non-NULL, which in turn 152 * ignore it for all cases but swap, where only page->private is 153 * of interest. When page_mapping() does go NULL, the entire 154 * call stack gracefully ignores the page and returns. 155 * -- wli 156 */ 157 smp_mb(); 158 mapping = page_mapping(page); 159 if (mapping && mapping->a_ops && mapping->a_ops->sync_page) 160 mapping->a_ops->sync_page(page); 161 io_schedule(); 162 return 0; 163} 164 165/** 166 * filemap_fdatawrite_range - start writeback against all of a mapping's 167 * dirty pages that lie within the byte offsets <start, end> 168 * @mapping: address space structure to write 169 * @start: offset in bytes where the range starts 170 * @end: offset in bytes where the range ends 171 * @sync_mode: enable synchronous operation 172 * 173 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as 174 * opposed to a regular memory * cleansing writeback. The difference between 175 * these two operations is that if a dirty page/buffer is encountered, it must 176 * be waited upon, and not just skipped over. 177 */ 178static int __filemap_fdatawrite_range(struct address_space *mapping, 179 loff_t start, loff_t end, int sync_mode) 180{ 181 int ret; 182 struct writeback_control wbc = { 183 .sync_mode = sync_mode, 184 .nr_to_write = mapping->nrpages * 2, 185 .start = start, 186 .end = end, 187 }; 188 189 if (!mapping_cap_writeback_dirty(mapping)) 190 return 0; 191 192 ret = do_writepages(mapping, &wbc); 193 return ret; 194} 195 196static inline int __filemap_fdatawrite(struct address_space *mapping, 197 int sync_mode) 198{ 199 return __filemap_fdatawrite_range(mapping, 0, 0, sync_mode); 200} 201 202int filemap_fdatawrite(struct address_space *mapping) 203{ 204 return __filemap_fdatawrite(mapping, WB_SYNC_ALL); 205} 206EXPORT_SYMBOL(filemap_fdatawrite); 207 208static int filemap_fdatawrite_range(struct address_space *mapping, 209 loff_t start, loff_t end) 210{ 211 return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL); 212} 213 214/* 215 * This is a mostly non-blocking flush. Not suitable for data-integrity 216 * purposes - I/O may not be started against all dirty pages. 217 */ 218int filemap_flush(struct address_space *mapping) 219{ 220 return __filemap_fdatawrite(mapping, WB_SYNC_NONE); 221} 222EXPORT_SYMBOL(filemap_flush); 223 224/* 225 * Wait for writeback to complete against pages indexed by start->end 226 * inclusive 227 */ 228static int wait_on_page_writeback_range(struct address_space *mapping, 229 pgoff_t start, pgoff_t end) 230{ 231 struct pagevec pvec; 232 int nr_pages; 233 int ret = 0; 234 pgoff_t index; 235 236 if (end < start) 237 return 0; 238 239 pagevec_init(&pvec, 0); 240 index = start; 241 while ((index <= end) && 242 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, 243 PAGECACHE_TAG_WRITEBACK, 244 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1)) != 0) { 245 unsigned i; 246 247 for (i = 0; i < nr_pages; i++) { 248 struct page *page = pvec.pages[i]; 249 250 /* until radix tree lookup accepts end_index */ 251 if (page->index > end) 252 continue; 253 254 wait_on_page_writeback(page); 255 if (PageError(page)) 256 ret = -EIO; 257 } 258 pagevec_release(&pvec); 259 cond_resched(); 260 } 261 262 /* Check for outstanding write errors */ 263 if (test_and_clear_bit(AS_ENOSPC, &mapping->flags)) 264 ret = -ENOSPC; 265 if (test_and_clear_bit(AS_EIO, &mapping->flags)) 266 ret = -EIO; 267 268 return ret; 269} 270 271/* 272 * Write and wait upon all the pages in the passed range. This is a "data 273 * integrity" operation. It waits upon in-flight writeout before starting and 274 * waiting upon new writeout. If there was an IO error, return it. 275 * 276 * We need to re-take i_sem during the generic_osync_inode list walk because 277 * it is otherwise livelockable. 278 */ 279int sync_page_range(struct inode *inode, struct address_space *mapping, 280 loff_t pos, size_t count) 281{ 282 pgoff_t start = pos >> PAGE_CACHE_SHIFT; 283 pgoff_t end = (pos + count - 1) >> PAGE_CACHE_SHIFT; 284 int ret; 285 286 if (!mapping_cap_writeback_dirty(mapping) || !count) 287 return 0; 288 ret = filemap_fdatawrite_range(mapping, pos, pos + count - 1); 289 if (ret == 0) { 290 down(&inode->i_sem); 291 ret = generic_osync_inode(inode, mapping, OSYNC_METADATA); 292 up(&inode->i_sem); 293 } 294 if (ret == 0) 295 ret = wait_on_page_writeback_range(mapping, start, end); 296 return ret; 297} 298EXPORT_SYMBOL(sync_page_range); 299 300/* 301 * Note: Holding i_sem across sync_page_range_nolock is not a good idea 302 * as it forces O_SYNC writers to different parts of the same file 303 * to be serialised right until io completion. 304 */ 305int sync_page_range_nolock(struct inode *inode, struct address_space *mapping, 306 loff_t pos, size_t count) 307{ 308 pgoff_t start = pos >> PAGE_CACHE_SHIFT; 309 pgoff_t end = (pos + count - 1) >> PAGE_CACHE_SHIFT; 310 int ret; 311 312 if (!mapping_cap_writeback_dirty(mapping) || !count) 313 return 0; 314 ret = filemap_fdatawrite_range(mapping, pos, pos + count - 1); 315 if (ret == 0) 316 ret = generic_osync_inode(inode, mapping, OSYNC_METADATA); 317 if (ret == 0) 318 ret = wait_on_page_writeback_range(mapping, start, end); 319 return ret; 320} 321EXPORT_SYMBOL(sync_page_range_nolock); 322 323/** 324 * filemap_fdatawait - walk the list of under-writeback pages of the given 325 * address space and wait for all of them. 326 * 327 * @mapping: address space structure to wait for 328 */ 329int filemap_fdatawait(struct address_space *mapping) 330{ 331 loff_t i_size = i_size_read(mapping->host); 332 333 if (i_size == 0) 334 return 0; 335 336 return wait_on_page_writeback_range(mapping, 0, 337 (i_size - 1) >> PAGE_CACHE_SHIFT); 338} 339EXPORT_SYMBOL(filemap_fdatawait); 340 341int filemap_write_and_wait(struct address_space *mapping) 342{ 343 int retval = 0; 344 345 if (mapping->nrpages) { 346 retval = filemap_fdatawrite(mapping); 347 if (retval == 0) 348 retval = filemap_fdatawait(mapping); 349 } 350 return retval; 351} 352 353int filemap_write_and_wait_range(struct address_space *mapping, 354 loff_t lstart, loff_t lend) 355{ 356 int retval = 0; 357 358 if (mapping->nrpages) { 359 retval = __filemap_fdatawrite_range(mapping, lstart, lend, 360 WB_SYNC_ALL); 361 if (retval == 0) 362 retval = wait_on_page_writeback_range(mapping, 363 lstart >> PAGE_CACHE_SHIFT, 364 lend >> PAGE_CACHE_SHIFT); 365 } 366 return retval; 367} 368 369/* 370 * This function is used to add newly allocated pagecache pages: 371 * the page is new, so we can just run SetPageLocked() against it. 372 * The other page state flags were set by rmqueue(). 373 * 374 * This function does not add the page to the LRU. The caller must do that. 375 */ 376int add_to_page_cache(struct page *page, struct address_space *mapping, 377 pgoff_t offset, int gfp_mask) 378{ 379 int error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM); 380 381 if (error == 0) { 382 write_lock_irq(&mapping->tree_lock); 383 error = radix_tree_insert(&mapping->page_tree, offset, page); 384 if (!error) { 385 page_cache_get(page); 386 SetPageLocked(page); 387 page->mapping = mapping; 388 page->index = offset; 389 mapping->nrpages++; 390 pagecache_acct(1); 391 } 392 write_unlock_irq(&mapping->tree_lock); 393 radix_tree_preload_end(); 394 } 395 return error; 396} 397 398EXPORT_SYMBOL(add_to_page_cache); 399 400int add_to_page_cache_lru(struct page *page, struct address_space *mapping, 401 pgoff_t offset, int gfp_mask) 402{ 403 int ret = add_to_page_cache(page, mapping, offset, gfp_mask); 404 if (ret == 0) 405 lru_cache_add(page); 406 return ret; 407} 408 409/* 410 * In order to wait for pages to become available there must be 411 * waitqueues associated with pages. By using a hash table of 412 * waitqueues where the bucket discipline is to maintain all 413 * waiters on the same queue and wake all when any of the pages 414 * become available, and for the woken contexts to check to be 415 * sure the appropriate page became available, this saves space 416 * at a cost of "thundering herd" phenomena during rare hash 417 * collisions. 418 */ 419static wait_queue_head_t *page_waitqueue(struct page *page) 420{ 421 const struct zone *zone = page_zone(page); 422 423 return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)]; 424} 425 426static inline void wake_up_page(struct page *page, int bit) 427{ 428 __wake_up_bit(page_waitqueue(page), &page->flags, bit); 429} 430 431void fastcall wait_on_page_bit(struct page *page, int bit_nr) 432{ 433 DEFINE_WAIT_BIT(wait, &page->flags, bit_nr); 434 435 if (test_bit(bit_nr, &page->flags)) 436 __wait_on_bit(page_waitqueue(page), &wait, sync_page, 437 TASK_UNINTERRUPTIBLE); 438} 439EXPORT_SYMBOL(wait_on_page_bit); 440 441/** 442 * unlock_page() - unlock a locked page 443 * 444 * @page: the page 445 * 446 * Unlocks the page and wakes up sleepers in ___wait_on_page_locked(). 447 * Also wakes sleepers in wait_on_page_writeback() because the wakeup 448 * mechananism between PageLocked pages and PageWriteback pages is shared. 449 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep. 450 * 451 * The first mb is necessary to safely close the critical section opened by the 452 * TestSetPageLocked(), the second mb is necessary to enforce ordering between 453 * the clear_bit and the read of the waitqueue (to avoid SMP races with a 454 * parallel wait_on_page_locked()). 455 */ 456void fastcall unlock_page(struct page *page) 457{ 458 smp_mb__before_clear_bit(); 459 if (!TestClearPageLocked(page)) 460 BUG(); 461 smp_mb__after_clear_bit(); 462 wake_up_page(page, PG_locked); 463} 464EXPORT_SYMBOL(unlock_page); 465 466/* 467 * End writeback against a page. 468 */ 469void end_page_writeback(struct page *page) 470{ 471 if (!TestClearPageReclaim(page) || rotate_reclaimable_page(page)) { 472 if (!test_clear_page_writeback(page)) 473 BUG(); 474 } 475 smp_mb__after_clear_bit(); 476 wake_up_page(page, PG_writeback); 477} 478EXPORT_SYMBOL(end_page_writeback); 479 480/* 481 * Get a lock on the page, assuming we need to sleep to get it. 482 * 483 * Ugly: running sync_page() in state TASK_UNINTERRUPTIBLE is scary. If some 484 * random driver's requestfn sets TASK_RUNNING, we could busywait. However 485 * chances are that on the second loop, the block layer's plug list is empty, 486 * so sync_page() will then return in state TASK_UNINTERRUPTIBLE. 487 */ 488void fastcall __lock_page(struct page *page) 489{ 490 DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); 491 492 __wait_on_bit_lock(page_waitqueue(page), &wait, sync_page, 493 TASK_UNINTERRUPTIBLE); 494} 495EXPORT_SYMBOL(__lock_page); 496 497/* 498 * a rather lightweight function, finding and getting a reference to a 499 * hashed page atomically. 500 */ 501struct page * find_get_page(struct address_space *mapping, unsigned long offset) 502{ 503 struct page *page; 504 505 read_lock_irq(&mapping->tree_lock); 506 page = radix_tree_lookup(&mapping->page_tree, offset); 507 if (page) 508 page_cache_get(page); 509 read_unlock_irq(&mapping->tree_lock); 510 return page; 511} 512 513EXPORT_SYMBOL(find_get_page); 514 515/* 516 * Same as above, but trylock it instead of incrementing the count. 517 */ 518struct page *find_trylock_page(struct address_space *mapping, unsigned long offset) 519{ 520 struct page *page; 521 522 read_lock_irq(&mapping->tree_lock); 523 page = radix_tree_lookup(&mapping->page_tree, offset); 524 if (page && TestSetPageLocked(page)) 525 page = NULL; 526 read_unlock_irq(&mapping->tree_lock); 527 return page; 528} 529 530EXPORT_SYMBOL(find_trylock_page); 531 532/** 533 * find_lock_page - locate, pin and lock a pagecache page 534 * 535 * @mapping: the address_space to search 536 * @offset: the page index 537 * 538 * Locates the desired pagecache page, locks it, increments its reference 539 * count and returns its address. 540 * 541 * Returns zero if the page was not present. find_lock_page() may sleep. 542 */ 543struct page *find_lock_page(struct address_space *mapping, 544 unsigned long offset) 545{ 546 struct page *page; 547 548 read_lock_irq(&mapping->tree_lock); 549repeat: 550 page = radix_tree_lookup(&mapping->page_tree, offset); 551 if (page) { 552 page_cache_get(page); 553 if (TestSetPageLocked(page)) { 554 read_unlock_irq(&mapping->tree_lock); 555 lock_page(page); 556 read_lock_irq(&mapping->tree_lock); 557 558 /* Has the page been truncated while we slept? */ 559 if (page->mapping != mapping || page->index != offset) { 560 unlock_page(page); 561 page_cache_release(page); 562 goto repeat; 563 } 564 } 565 } 566 read_unlock_irq(&mapping->tree_lock); 567 return page; 568} 569 570EXPORT_SYMBOL(find_lock_page); 571 572/** 573 * find_or_create_page - locate or add a pagecache page 574 * 575 * @mapping: the page's address_space 576 * @index: the page's index into the mapping 577 * @gfp_mask: page allocation mode 578 * 579 * Locates a page in the pagecache. If the page is not present, a new page 580 * is allocated using @gfp_mask and is added to the pagecache and to the VM's 581 * LRU list. The returned page is locked and has its reference count 582 * incremented. 583 * 584 * find_or_create_page() may sleep, even if @gfp_flags specifies an atomic 585 * allocation! 586 * 587 * find_or_create_page() returns the desired page's address, or zero on 588 * memory exhaustion. 589 */ 590struct page *find_or_create_page(struct address_space *mapping, 591 unsigned long index, unsigned int gfp_mask) 592{ 593 struct page *page, *cached_page = NULL; 594 int err; 595repeat: 596 page = find_lock_page(mapping, index); 597 if (!page) { 598 if (!cached_page) { 599 cached_page = alloc_page(gfp_mask); 600 if (!cached_page) 601 return NULL; 602 } 603 err = add_to_page_cache_lru(cached_page, mapping, 604 index, gfp_mask); 605 if (!err) { 606 page = cached_page; 607 cached_page = NULL; 608 } else if (err == -EEXIST) 609 goto repeat; 610 } 611 if (cached_page) 612 page_cache_release(cached_page); 613 return page; 614} 615 616EXPORT_SYMBOL(find_or_create_page); 617 618/** 619 * find_get_pages - gang pagecache lookup 620 * @mapping: The address_space to search 621 * @start: The starting page index 622 * @nr_pages: The maximum number of pages 623 * @pages: Where the resulting pages are placed 624 * 625 * find_get_pages() will search for and return a group of up to 626 * @nr_pages pages in the mapping. The pages are placed at @pages. 627 * find_get_pages() takes a reference against the returned pages. 628 * 629 * The search returns a group of mapping-contiguous pages with ascending 630 * indexes. There may be holes in the indices due to not-present pages. 631 * 632 * find_get_pages() returns the number of pages which were found. 633 */ 634unsigned find_get_pages(struct address_space *mapping, pgoff_t start, 635 unsigned int nr_pages, struct page **pages) 636{ 637 unsigned int i; 638 unsigned int ret; 639 640 read_lock_irq(&mapping->tree_lock); 641 ret = radix_tree_gang_lookup(&mapping->page_tree, 642 (void **)pages, start, nr_pages); 643 for (i = 0; i < ret; i++) 644 page_cache_get(pages[i]); 645 read_unlock_irq(&mapping->tree_lock); 646 return ret; 647} 648 649/* 650 * Like find_get_pages, except we only return pages which are tagged with 651 * `tag'. We update *index to index the next page for the traversal. 652 */ 653unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index, 654 int tag, unsigned int nr_pages, struct page **pages) 655{ 656 unsigned int i; 657 unsigned int ret; 658 659 read_lock_irq(&mapping->tree_lock); 660 ret = radix_tree_gang_lookup_tag(&mapping->page_tree, 661 (void **)pages, *index, nr_pages, tag); 662 for (i = 0; i < ret; i++) 663 page_cache_get(pages[i]); 664 if (ret) 665 *index = pages[ret - 1]->index + 1; 666 read_unlock_irq(&mapping->tree_lock); 667 return ret; 668} 669 670/* 671 * Same as grab_cache_page, but do not wait if the page is unavailable. 672 * This is intended for speculative data generators, where the data can 673 * be regenerated if the page couldn't be grabbed. This routine should 674 * be safe to call while holding the lock for another page. 675 * 676 * Clear __GFP_FS when allocating the page to avoid recursion into the fs 677 * and deadlock against the caller's locked page. 678 */ 679struct page * 680grab_cache_page_nowait(struct address_space *mapping, unsigned long index) 681{ 682 struct page *page = find_get_page(mapping, index); 683 unsigned int gfp_mask; 684 685 if (page) { 686 if (!TestSetPageLocked(page)) 687 return page; 688 page_cache_release(page); 689 return NULL; 690 } 691 gfp_mask = mapping_gfp_mask(mapping) & ~__GFP_FS; 692 page = alloc_pages(gfp_mask, 0); 693 if (page && add_to_page_cache_lru(page, mapping, index, gfp_mask)) { 694 page_cache_release(page); 695 page = NULL; 696 } 697 return page; 698} 699 700EXPORT_SYMBOL(grab_cache_page_nowait); 701 702/* 703 * This is a generic file read routine, and uses the 704 * mapping->a_ops->readpage() function for the actual low-level 705 * stuff. 706 * 707 * This is really ugly. But the goto's actually try to clarify some 708 * of the logic when it comes to error handling etc. 709 * 710 * Note the struct file* is only passed for the use of readpage. It may be 711 * NULL. 712 */ 713void do_generic_mapping_read(struct address_space *mapping, 714 struct file_ra_state *_ra, 715 struct file *filp, 716 loff_t *ppos, 717 read_descriptor_t *desc, 718 read_actor_t actor) 719{ 720 struct inode *inode = mapping->host; 721 unsigned long index; 722 unsigned long end_index; 723 unsigned long offset; 724 unsigned long last_index; 725 unsigned long next_index; 726 unsigned long prev_index; 727 loff_t isize; 728 struct page *cached_page; 729 int error; 730 struct file_ra_state ra = *_ra; 731 732 cached_page = NULL; 733 index = *ppos >> PAGE_CACHE_SHIFT; 734 next_index = index; 735 prev_index = ra.prev_page; 736 last_index = (*ppos + desc->count + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT; 737 offset = *ppos & ~PAGE_CACHE_MASK; 738 739 isize = i_size_read(inode); 740 if (!isize) 741 goto out; 742 743 end_index = (isize - 1) >> PAGE_CACHE_SHIFT; 744 for (;;) { 745 struct page *page; 746 unsigned long nr, ret; 747 748 /* nr is the maximum number of bytes to copy from this page */ 749 nr = PAGE_CACHE_SIZE; 750 if (index >= end_index) { 751 if (index > end_index) 752 goto out; 753 nr = ((isize - 1) & ~PAGE_CACHE_MASK) + 1; 754 if (nr <= offset) { 755 goto out; 756 } 757 } 758 nr = nr - offset; 759 760 cond_resched(); 761 if (index == next_index) 762 next_index = page_cache_readahead(mapping, &ra, filp, 763 index, last_index - index); 764 765find_page: 766 page = find_get_page(mapping, index); 767 if (unlikely(page == NULL)) { 768 handle_ra_miss(mapping, &ra, index); 769 goto no_cached_page; 770 } 771 if (!PageUptodate(page)) 772 goto page_not_up_to_date; 773page_ok: 774 775 /* If users can be writing to this page using arbitrary 776 * virtual addresses, take care about potential aliasing 777 * before reading the page on the kernel side. 778 */ 779 if (mapping_writably_mapped(mapping)) 780 flush_dcache_page(page); 781 782 /* 783 * When (part of) the same page is read multiple times 784 * in succession, only mark it as accessed the first time. 785 */ 786 if (prev_index != index) 787 mark_page_accessed(page); 788 prev_index = index; 789 790 /* 791 * Ok, we have the page, and it's up-to-date, so 792 * now we can copy it to user space... 793 * 794 * The actor routine returns how many bytes were actually used.. 795 * NOTE! This may not be the same as how much of a user buffer 796 * we filled up (we may be padding etc), so we can only update 797 * "pos" here (the actor routine has to update the user buffer 798 * pointers and the remaining count). 799 */ 800 ret = actor(desc, page, offset, nr); 801 offset += ret; 802 index += offset >> PAGE_CACHE_SHIFT; 803 offset &= ~PAGE_CACHE_MASK; 804 805 page_cache_release(page); 806 if (ret == nr && desc->count) 807 continue; 808 goto out; 809 810page_not_up_to_date: 811 /* Get exclusive access to the page ... */ 812 lock_page(page); 813 814 /* Did it get unhashed before we got the lock? */ 815 if (!page->mapping) { 816 unlock_page(page); 817 page_cache_release(page); 818 continue; 819 } 820 821 /* Did somebody else fill it already? */ 822 if (PageUptodate(page)) { 823 unlock_page(page); 824 goto page_ok; 825 } 826 827readpage: 828 /* Start the actual read. The read will unlock the page. */ 829 error = mapping->a_ops->readpage(filp, page); 830 831 if (unlikely(error)) 832 goto readpage_error; 833 834 if (!PageUptodate(page)) { 835 lock_page(page); 836 if (!PageUptodate(page)) { 837 if (page->mapping == NULL) { 838 /* 839 * invalidate_inode_pages got it 840 */ 841 unlock_page(page); 842 page_cache_release(page); 843 goto find_page; 844 } 845 unlock_page(page); 846 error = -EIO; 847 goto readpage_error; 848 } 849 unlock_page(page); 850 } 851 852 /* 853 * i_size must be checked after we have done ->readpage. 854 * 855 * Checking i_size after the readpage allows us to calculate 856 * the correct value for "nr", which means the zero-filled 857 * part of the page is not copied back to userspace (unless 858 * another truncate extends the file - this is desired though). 859 */ 860 isize = i_size_read(inode); 861 end_index = (isize - 1) >> PAGE_CACHE_SHIFT; 862 if (unlikely(!isize || index > end_index)) { 863 page_cache_release(page); 864 goto out; 865 } 866 867 /* nr is the maximum number of bytes to copy from this page */ 868 nr = PAGE_CACHE_SIZE; 869 if (index == end_index) { 870 nr = ((isize - 1) & ~PAGE_CACHE_MASK) + 1; 871 if (nr <= offset) { 872 page_cache_release(page); 873 goto out; 874 } 875 } 876 nr = nr - offset; 877 goto page_ok; 878 879readpage_error: 880 /* UHHUH! A synchronous read error occurred. Report it */ 881 desc->error = error; 882 page_cache_release(page); 883 goto out; 884 885no_cached_page: 886 /* 887 * Ok, it wasn't cached, so we need to create a new 888 * page.. 889 */ 890 if (!cached_page) { 891 cached_page = page_cache_alloc_cold(mapping); 892 if (!cached_page) { 893 desc->error = -ENOMEM; 894 goto out; 895 } 896 } 897 error = add_to_page_cache_lru(cached_page, mapping, 898 index, GFP_KERNEL); 899 if (error) { 900 if (error == -EEXIST) 901 goto find_page; 902 desc->error = error; 903 goto out; 904 } 905 page = cached_page; 906 cached_page = NULL; 907 goto readpage; 908 } 909 910out: 911 *_ra = ra; 912 913 *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset; 914 if (cached_page) 915 page_cache_release(cached_page); 916 if (filp) 917 file_accessed(filp); 918} 919 920EXPORT_SYMBOL(do_generic_mapping_read); 921 922int file_read_actor(read_descriptor_t *desc, struct page *page, 923 unsigned long offset, unsigned long size) 924{ 925 char *kaddr; 926 unsigned long left, count = desc->count; 927 928 if (size > count) 929 size = count; 930 931 /* 932 * Faults on the destination of a read are common, so do it before 933 * taking the kmap. 934 */ 935 if (!fault_in_pages_writeable(desc->arg.buf, size)) { 936 kaddr = kmap_atomic(page, KM_USER0); 937 left = __copy_to_user_inatomic(desc->arg.buf, 938 kaddr + offset, size); 939 kunmap_atomic(kaddr, KM_USER0); 940 if (left == 0) 941 goto success; 942 } 943 944 /* Do it the slow way */ 945 kaddr = kmap(page); 946 left = __copy_to_user(desc->arg.buf, kaddr + offset, size); 947 kunmap(page); 948 949 if (left) { 950 size -= left; 951 desc->error = -EFAULT; 952 } 953success: 954 desc->count = count - size; 955 desc->written += size; 956 desc->arg.buf += size; 957 return size; 958} 959 960/* 961 * This is the "read()" routine for all filesystems 962 * that can use the page cache directly. 963 */ 964ssize_t 965__generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov, 966 unsigned long nr_segs, loff_t *ppos) 967{ 968 struct file *filp = iocb->ki_filp; 969 ssize_t retval; 970 unsigned long seg; 971 size_t count; 972 973 count = 0; 974 for (seg = 0; seg < nr_segs; seg++) { 975 const struct iovec *iv = &iov[seg]; 976 977 /* 978 * If any segment has a negative length, or the cumulative 979 * length ever wraps negative then return -EINVAL. 980 */ 981 count += iv->iov_len; 982 if (unlikely((ssize_t)(count|iv->iov_len) < 0)) 983 return -EINVAL; 984 if (access_ok(VERIFY_WRITE, iv->iov_base, iv->iov_len)) 985 continue; 986 if (seg == 0) 987 return -EFAULT; 988 nr_segs = seg; 989 count -= iv->iov_len; /* This segment is no good */ 990 break; 991 } 992 993 /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */ 994 if (filp->f_flags & O_DIRECT) { 995 loff_t pos = *ppos, size; 996 struct address_space *mapping; 997 struct inode *inode; 998 999 mapping = filp->f_mapping; 1000 inode = mapping->host; 1001 retval = 0; 1002 if (!count) 1003 goto out; /* skip atime */ 1004 size = i_size_read(inode); 1005 if (pos < size) { 1006 retval = generic_file_direct_IO(READ, iocb, 1007 iov, pos, nr_segs); 1008 if (retval > 0 && !is_sync_kiocb(iocb)) 1009 retval = -EIOCBQUEUED; 1010 if (retval > 0) 1011 *ppos = pos + retval; 1012 } 1013 file_accessed(filp); 1014 goto out; 1015 } 1016 1017 retval = 0; 1018 if (count) { 1019 for (seg = 0; seg < nr_segs; seg++) { 1020 read_descriptor_t desc; 1021 1022 desc.written = 0; 1023 desc.arg.buf = iov[seg].iov_base; 1024 desc.count = iov[seg].iov_len; 1025 if (desc.count == 0) 1026 continue; 1027 desc.error = 0; 1028 do_generic_file_read(filp,ppos,&desc,file_read_actor); 1029 retval += desc.written; 1030 if (!retval) { 1031 retval = desc.error; 1032 break; 1033 } 1034 } 1035 } 1036out: 1037 return retval; 1038} 1039 1040EXPORT_SYMBOL(__generic_file_aio_read); 1041 1042ssize_t 1043generic_file_aio_read(struct kiocb *iocb, char __user *buf, size_t count, loff_t pos) 1044{ 1045 struct iovec local_iov = { .iov_base = buf, .iov_len = count }; 1046 1047 BUG_ON(iocb->ki_pos != pos); 1048 return __generic_file_aio_read(iocb, &local_iov, 1, &iocb->ki_pos); 1049} 1050 1051EXPORT_SYMBOL(generic_file_aio_read); 1052 1053ssize_t 1054generic_file_read(struct file *filp, char __user *buf, size_t count, loff_t *ppos) 1055{ 1056 struct iovec local_iov = { .iov_base = buf, .iov_len = count }; 1057 struct kiocb kiocb; 1058 ssize_t ret; 1059 1060 init_sync_kiocb(&kiocb, filp); 1061 ret = __generic_file_aio_read(&kiocb, &local_iov, 1, ppos); 1062 if (-EIOCBQUEUED == ret) 1063 ret = wait_on_sync_kiocb(&kiocb); 1064 return ret; 1065} 1066 1067EXPORT_SYMBOL(generic_file_read); 1068 1069int file_send_actor(read_descriptor_t * desc, struct page *page, unsigned long offset, unsigned long size) 1070{ 1071 ssize_t written; 1072 unsigned long count = desc->count; 1073 struct file *file = desc->arg.data; 1074 1075 if (size > count) 1076 size = count; 1077 1078 written = file->f_op->sendpage(file, page, offset, 1079 size, &file->f_pos, size<count); 1080 if (written < 0) { 1081 desc->error = written; 1082 written = 0; 1083 } 1084 desc->count = count - written; 1085 desc->written += written; 1086 return written; 1087} 1088 1089ssize_t generic_file_sendfile(struct file *in_file, loff_t *ppos, 1090 size_t count, read_actor_t actor, void *target) 1091{ 1092 read_descriptor_t desc; 1093 1094 if (!count) 1095 return 0; 1096 1097 desc.written = 0; 1098 desc.count = count; 1099 desc.arg.data = target; 1100 desc.error = 0; 1101 1102 do_generic_file_read(in_file, ppos, &desc, actor); 1103 if (desc.written) 1104 return desc.written; 1105 return desc.error; 1106} 1107 1108EXPORT_SYMBOL(generic_file_sendfile); 1109 1110static ssize_t 1111do_readahead(struct address_space *mapping, struct file *filp, 1112 unsigned long index, unsigned long nr) 1113{ 1114 if (!mapping || !mapping->a_ops || !mapping->a_ops->readpage) 1115 return -EINVAL; 1116 1117 force_page_cache_readahead(mapping, filp, index, 1118 max_sane_readahead(nr)); 1119 return 0; 1120} 1121 1122asmlinkage ssize_t sys_readahead(int fd, loff_t offset, size_t count) 1123{ 1124 ssize_t ret; 1125 struct file *file; 1126 1127 ret = -EBADF; 1128 file = fget(fd); 1129 if (file) { 1130 if (file->f_mode & FMODE_READ) { 1131 struct address_space *mapping = file->f_mapping; 1132 unsigned long start = offset >> PAGE_CACHE_SHIFT; 1133 unsigned long end = (offset + count - 1) >> PAGE_CACHE_SHIFT; 1134 unsigned long len = end - start + 1; 1135 ret = do_readahead(mapping, file, start, len); 1136 } 1137 fput(file); 1138 } 1139 return ret; 1140} 1141 1142#ifdef CONFIG_MMU 1143/* 1144 * This adds the requested page to the page cache if it isn't already there, 1145 * and schedules an I/O to read in its contents from disk. 1146 */ 1147static int FASTCALL(page_cache_read(struct file * file, unsigned long offset)); 1148static int fastcall page_cache_read(struct file * file, unsigned long offset) 1149{ 1150 struct address_space *mapping = file->f_mapping; 1151 struct page *page; 1152 int error; 1153 1154 page = page_cache_alloc_cold(mapping); 1155 if (!page) 1156 return -ENOMEM; 1157 1158 error = add_to_page_cache_lru(page, mapping, offset, GFP_KERNEL); 1159 if (!error) { 1160 error = mapping->a_ops->readpage(file, page); 1161 page_cache_release(page); 1162 return error; 1163 } 1164 1165 /* 1166 * We arrive here in the unlikely event that someone 1167 * raced with us and added our page to the cache first 1168 * or we are out of memory for radix-tree nodes. 1169 */ 1170 page_cache_release(page); 1171 return error == -EEXIST ? 0 : error; 1172} 1173 1174#define MMAP_LOTSAMISS (100) 1175 1176/* 1177 * filemap_nopage() is invoked via the vma operations vector for a 1178 * mapped memory region to read in file data during a page fault. 1179 * 1180 * The goto's are kind of ugly, but this streamlines the normal case of having 1181 * it in the page cache, and handles the special cases reasonably without 1182 * having a lot of duplicated code. 1183 */ 1184struct page *filemap_nopage(struct vm_area_struct *area, 1185 unsigned long address, int *type) 1186{ 1187 int error; 1188 struct file *file = area->vm_file; 1189 struct address_space *mapping = file->f_mapping; 1190 struct file_ra_state *ra = &file->f_ra; 1191 struct inode *inode = mapping->host; 1192 struct page *page; 1193 unsigned long size, pgoff; 1194 int did_readaround = 0, majmin = VM_FAULT_MINOR; 1195 1196 pgoff = ((address-area->vm_start) >> PAGE_CACHE_SHIFT) + area->vm_pgoff; 1197 1198retry_all: 1199 size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 1200 if (pgoff >= size) 1201 goto outside_data_content; 1202 1203 /* If we don't want any read-ahead, don't bother */ 1204 if (VM_RandomReadHint(area)) 1205 goto no_cached_page; 1206 1207 /* 1208 * The readahead code wants to be told about each and every page 1209 * so it can build and shrink its windows appropriately 1210 * 1211 * For sequential accesses, we use the generic readahead logic. 1212 */ 1213 if (VM_SequentialReadHint(area)) 1214 page_cache_readahead(mapping, ra, file, pgoff, 1); 1215 1216 /* 1217 * Do we have something in the page cache already? 1218 */ 1219retry_find: 1220 page = find_get_page(mapping, pgoff); 1221 if (!page) { 1222 unsigned long ra_pages; 1223 1224 if (VM_SequentialReadHint(area)) { 1225 handle_ra_miss(mapping, ra, pgoff); 1226 goto no_cached_page; 1227 } 1228 ra->mmap_miss++; 1229 1230 /* 1231 * Do we miss much more than hit in this file? If so, 1232 * stop bothering with read-ahead. It will only hurt. 1233 */ 1234 if (ra->mmap_miss > ra->mmap_hit + MMAP_LOTSAMISS) 1235 goto no_cached_page; 1236 1237 /* 1238 * To keep the pgmajfault counter straight, we need to 1239 * check did_readaround, as this is an inner loop. 1240 */ 1241 if (!did_readaround) { 1242 majmin = VM_FAULT_MAJOR; 1243 inc_page_state(pgmajfault); 1244 } 1245 did_readaround = 1; 1246 ra_pages = max_sane_readahead(file->f_ra.ra_pages); 1247 if (ra_pages) { 1248 pgoff_t start = 0; 1249 1250 if (pgoff > ra_pages / 2) 1251 start = pgoff - ra_pages / 2; 1252 do_page_cache_readahead(mapping, file, start, ra_pages); 1253 } 1254 page = find_get_page(mapping, pgoff); 1255 if (!page) 1256 goto no_cached_page; 1257 } 1258 1259 if (!did_readaround) 1260 ra->mmap_hit++; 1261 1262 /* 1263 * Ok, found a page in the page cache, now we need to check 1264 * that it's up-to-date. 1265 */ 1266 if (!PageUptodate(page)) 1267 goto page_not_uptodate; 1268 1269success: 1270 /* 1271 * Found the page and have a reference on it. 1272 */ 1273 mark_page_accessed(page); 1274 if (type) 1275 *type = majmin; 1276 return page; 1277 1278outside_data_content: 1279 /* 1280 * An external ptracer can access pages that normally aren't 1281 * accessible.. 1282 */ 1283 if (area->vm_mm == current->mm) 1284 return NULL; 1285 /* Fall through to the non-read-ahead case */ 1286no_cached_page: 1287 /* 1288 * We're only likely to ever get here if MADV_RANDOM is in 1289 * effect. 1290 */ 1291 error = page_cache_read(file, pgoff); 1292 grab_swap_token(); 1293 1294 /* 1295 * The page we want has now been added to the page cache. 1296 * In the unlikely event that someone removed it in the 1297 * meantime, we'll just come back here and read it again. 1298 */ 1299 if (error >= 0) 1300 goto retry_find; 1301 1302 /* 1303 * An error return from page_cache_read can result if the 1304 * system is low on memory, or a problem occurs while trying 1305 * to schedule I/O. 1306 */ 1307 if (error == -ENOMEM) 1308 return NOPAGE_OOM; 1309 return NULL; 1310 1311page_not_uptodate: 1312 if (!did_readaround) { 1313 majmin = VM_FAULT_MAJOR; 1314 inc_page_state(pgmajfault); 1315 } 1316 lock_page(page); 1317 1318 /* Did it get unhashed while we waited for it? */ 1319 if (!page->mapping) { 1320 unlock_page(page); 1321 page_cache_release(page); 1322 goto retry_all; 1323 } 1324 1325 /* Did somebody else get it up-to-date? */ 1326 if (PageUptodate(page)) { 1327 unlock_page(page); 1328 goto success; 1329 } 1330 1331 if (!mapping->a_ops->readpage(file, page)) { 1332 wait_on_page_locked(page); 1333 if (PageUptodate(page)) 1334 goto success; 1335 } 1336 1337 /* 1338 * Umm, take care of errors if the page isn't up-to-date. 1339 * Try to re-read it _once_. We do this synchronously, 1340 * because there really aren't any performance issues here 1341 * and we need to check for errors. 1342 */ 1343 lock_page(page); 1344 1345 /* Somebody truncated the page on us? */ 1346 if (!page->mapping) { 1347 unlock_page(page); 1348 page_cache_release(page); 1349 goto retry_all; 1350 } 1351 1352 /* Somebody else successfully read it in? */ 1353 if (PageUptodate(page)) { 1354 unlock_page(page); 1355 goto success; 1356 } 1357 ClearPageError(page); 1358 if (!mapping->a_ops->readpage(file, page)) { 1359 wait_on_page_locked(page); 1360 if (PageUptodate(page)) 1361 goto success; 1362 } 1363 1364 /* 1365 * Things didn't work out. Return zero to tell the 1366 * mm layer so, possibly freeing the page cache page first. 1367 */ 1368 page_cache_release(page); 1369 return NULL; 1370} 1371 1372EXPORT_SYMBOL(filemap_nopage); 1373 1374static struct page * filemap_getpage(struct file *file, unsigned long pgoff, 1375 int nonblock) 1376{ 1377 struct address_space *mapping = file->f_mapping; 1378 struct page *page; 1379 int error; 1380 1381 /* 1382 * Do we have something in the page cache already? 1383 */ 1384retry_find: 1385 page = find_get_page(mapping, pgoff); 1386 if (!page) { 1387 if (nonblock) 1388 return NULL; 1389 goto no_cached_page; 1390 } 1391 1392 /* 1393 * Ok, found a page in the page cache, now we need to check 1394 * that it's up-to-date. 1395 */ 1396 if (!PageUptodate(page)) { 1397 if (nonblock) { 1398 page_cache_release(page); 1399 return NULL; 1400 } 1401 goto page_not_uptodate; 1402 } 1403 1404success: 1405 /* 1406 * Found the page and have a reference on it. 1407 */ 1408 mark_page_accessed(page); 1409 return page; 1410 1411no_cached_page: 1412 error = page_cache_read(file, pgoff); 1413 1414 /* 1415 * The page we want has now been added to the page cache. 1416 * In the unlikely event that someone removed it in the 1417 * meantime, we'll just come back here and read it again. 1418 */ 1419 if (error >= 0) 1420 goto retry_find; 1421 1422 /* 1423 * An error return from page_cache_read can result if the 1424 * system is low on memory, or a problem occurs while trying 1425 * to schedule I/O. 1426 */ 1427 return NULL; 1428 1429page_not_uptodate: 1430 lock_page(page); 1431 1432 /* Did it get unhashed while we waited for it? */ 1433 if (!page->mapping) { 1434 unlock_page(page); 1435 goto err; 1436 } 1437 1438 /* Did somebody else get it up-to-date? */ 1439 if (PageUptodate(page)) { 1440 unlock_page(page); 1441 goto success; 1442 } 1443 1444 if (!mapping->a_ops->readpage(file, page)) { 1445 wait_on_page_locked(page); 1446 if (PageUptodate(page)) 1447 goto success; 1448 } 1449 1450 /* 1451 * Umm, take care of errors if the page isn't up-to-date. 1452 * Try to re-read it _once_. We do this synchronously, 1453 * because there really aren't any performance issues here 1454 * and we need to check for errors. 1455 */ 1456 lock_page(page); 1457 1458 /* Somebody truncated the page on us? */ 1459 if (!page->mapping) { 1460 unlock_page(page); 1461 goto err; 1462 } 1463 /* Somebody else successfully read it in? */ 1464 if (PageUptodate(page)) { 1465 unlock_page(page); 1466 goto success; 1467 } 1468 1469 ClearPageError(page); 1470 if (!mapping->a_ops->readpage(file, page)) { 1471 wait_on_page_locked(page); 1472 if (PageUptodate(page)) 1473 goto success; 1474 } 1475 1476 /* 1477 * Things didn't work out. Return zero to tell the 1478 * mm layer so, possibly freeing the page cache page first. 1479 */ 1480err: 1481 page_cache_release(page); 1482 1483 return NULL; 1484} 1485 1486int filemap_populate(struct vm_area_struct *vma, unsigned long addr, 1487 unsigned long len, pgprot_t prot, unsigned long pgoff, 1488 int nonblock) 1489{ 1490 struct file *file = vma->vm_file; 1491 struct address_space *mapping = file->f_mapping; 1492 struct inode *inode = mapping->host; 1493 unsigned long size; 1494 struct mm_struct *mm = vma->vm_mm; 1495 struct page *page; 1496 int err; 1497 1498 if (!nonblock) 1499 force_page_cache_readahead(mapping, vma->vm_file, 1500 pgoff, len >> PAGE_CACHE_SHIFT); 1501 1502repeat: 1503 size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 1504 if (pgoff + (len >> PAGE_CACHE_SHIFT) > size) 1505 return -EINVAL; 1506 1507 page = filemap_getpage(file, pgoff, nonblock); 1508 if (!page && !nonblock) 1509 return -ENOMEM; 1510 if (page) { 1511 err = install_page(mm, vma, addr, page, prot); 1512 if (err) { 1513 page_cache_release(page); 1514 return err; 1515 } 1516 } else { 1517 err = install_file_pte(mm, vma, addr, pgoff, prot); 1518 if (err) 1519 return err; 1520 } 1521 1522 len -= PAGE_SIZE; 1523 addr += PAGE_SIZE; 1524 pgoff++; 1525 if (len) 1526 goto repeat; 1527 1528 return 0; 1529} 1530 1531struct vm_operations_struct generic_file_vm_ops = { 1532 .nopage = filemap_nopage, 1533 .populate = filemap_populate, 1534}; 1535 1536/* This is used for a general mmap of a disk file */ 1537 1538int generic_file_mmap(struct file * file, struct vm_area_struct * vma) 1539{ 1540 struct address_space *mapping = file->f_mapping; 1541 1542 if (!mapping->a_ops->readpage) 1543 return -ENOEXEC; 1544 file_accessed(file); 1545 vma->vm_ops = &generic_file_vm_ops; 1546 return 0; 1547} 1548EXPORT_SYMBOL(filemap_populate); 1549 1550/* 1551 * This is for filesystems which do not implement ->writepage. 1552 */ 1553int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma) 1554{ 1555 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) 1556 return -EINVAL; 1557 return generic_file_mmap(file, vma); 1558} 1559#else 1560int generic_file_mmap(struct file * file, struct vm_area_struct * vma) 1561{ 1562 return -ENOSYS; 1563} 1564int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma) 1565{ 1566 return -ENOSYS; 1567} 1568#endif /* CONFIG_MMU */ 1569 1570EXPORT_SYMBOL(generic_file_mmap); 1571EXPORT_SYMBOL(generic_file_readonly_mmap); 1572 1573static inline struct page *__read_cache_page(struct address_space *mapping, 1574 unsigned long index, 1575 int (*filler)(void *,struct page*), 1576 void *data) 1577{ 1578 struct page *page, *cached_page = NULL; 1579 int err; 1580repeat: 1581 page = find_get_page(mapping, index); 1582 if (!page) { 1583 if (!cached_page) { 1584 cached_page = page_cache_alloc_cold(mapping); 1585 if (!cached_page) 1586 return ERR_PTR(-ENOMEM); 1587 } 1588 err = add_to_page_cache_lru(cached_page, mapping, 1589 index, GFP_KERNEL); 1590 if (err == -EEXIST) 1591 goto repeat; 1592 if (err < 0) { 1593 /* Presumably ENOMEM for radix tree node */ 1594 page_cache_release(cached_page); 1595 return ERR_PTR(err); 1596 } 1597 page = cached_page; 1598 cached_page = NULL; 1599 err = filler(data, page); 1600 if (err < 0) { 1601 page_cache_release(page); 1602 page = ERR_PTR(err); 1603 } 1604 } 1605 if (cached_page) 1606 page_cache_release(cached_page); 1607 return page; 1608} 1609 1610/* 1611 * Read into the page cache. If a page already exists, 1612 * and PageUptodate() is not set, try to fill the page. 1613 */ 1614struct page *read_cache_page(struct address_space *mapping, 1615 unsigned long index, 1616 int (*filler)(void *,struct page*), 1617 void *data) 1618{ 1619 struct page *page; 1620 int err; 1621 1622retry: 1623 page = __read_cache_page(mapping, index, filler, data); 1624 if (IS_ERR(page)) 1625 goto out; 1626 mark_page_accessed(page); 1627 if (PageUptodate(page)) 1628 goto out; 1629 1630 lock_page(page); 1631 if (!page->mapping) { 1632 unlock_page(page); 1633 page_cache_release(page); 1634 goto retry; 1635 } 1636 if (PageUptodate(page)) { 1637 unlock_page(page); 1638 goto out; 1639 } 1640 err = filler(data, page); 1641 if (err < 0) { 1642 page_cache_release(page); 1643 page = ERR_PTR(err); 1644 } 1645 out: 1646 return page; 1647} 1648 1649EXPORT_SYMBOL(read_cache_page); 1650 1651/* 1652 * If the page was newly created, increment its refcount and add it to the 1653 * caller's lru-buffering pagevec. This function is specifically for 1654 * generic_file_write(). 1655 */ 1656static inline struct page * 1657__grab_cache_page(struct address_space *mapping, unsigned long index, 1658 struct page **cached_page, struct pagevec *lru_pvec) 1659{ 1660 int err; 1661 struct page *page; 1662repeat: 1663 page = find_lock_page(mapping, index); 1664 if (!page) { 1665 if (!*cached_page) { 1666 *cached_page = page_cache_alloc(mapping); 1667 if (!*cached_page) 1668 return NULL; 1669 } 1670 err = add_to_page_cache(*cached_page, mapping, 1671 index, GFP_KERNEL); 1672 if (err == -EEXIST) 1673 goto repeat; 1674 if (err == 0) { 1675 page = *cached_page; 1676 page_cache_get(page); 1677 if (!pagevec_add(lru_pvec, page)) 1678 __pagevec_lru_add(lru_pvec); 1679 *cached_page = NULL; 1680 } 1681 } 1682 return page; 1683} 1684 1685/* 1686 * The logic we want is 1687 * 1688 * if suid or (sgid and xgrp) 1689 * remove privs 1690 */ 1691int remove_suid(struct dentry *dentry) 1692{ 1693 mode_t mode = dentry->d_inode->i_mode; 1694 int kill = 0; 1695 int result = 0; 1696 1697 /* suid always must be killed */ 1698 if (unlikely(mode & S_ISUID)) 1699 kill = ATTR_KILL_SUID; 1700 1701 /* 1702 * sgid without any exec bits is just a mandatory locking mark; leave 1703 * it alone. If some exec bits are set, it's a real sgid; kill it. 1704 */ 1705 if (unlikely((mode & S_ISGID) && (mode & S_IXGRP))) 1706 kill |= ATTR_KILL_SGID; 1707 1708 if (unlikely(kill && !capable(CAP_FSETID))) { 1709 struct iattr newattrs; 1710 1711 newattrs.ia_valid = ATTR_FORCE | kill; 1712 result = notify_change(dentry, &newattrs); 1713 } 1714 return result; 1715} 1716EXPORT_SYMBOL(remove_suid); 1717 1718size_t 1719__filemap_copy_from_user_iovec(char *vaddr, 1720 const struct iovec *iov, size_t base, size_t bytes) 1721{ 1722 size_t copied = 0, left = 0; 1723 1724 while (bytes) { 1725 char __user *buf = iov->iov_base + base; 1726 int copy = min(bytes, iov->iov_len - base); 1727 1728 base = 0; 1729 left = __copy_from_user_inatomic(vaddr, buf, copy); 1730 copied += copy; 1731 bytes -= copy; 1732 vaddr += copy; 1733 iov++; 1734 1735 if (unlikely(left)) { 1736 /* zero the rest of the target like __copy_from_user */ 1737 if (bytes) 1738 memset(vaddr, 0, bytes); 1739 break; 1740 } 1741 } 1742 return copied - left; 1743} 1744 1745/* 1746 * Performs necessary checks before doing a write 1747 * 1748 * Can adjust writing position aor amount of bytes to write. 1749 * Returns appropriate error code that caller should return or 1750 * zero in case that write should be allowed. 1751 */ 1752inline int generic_write_checks(struct file *file, loff_t *pos, size_t *count, int isblk) 1753{ 1754 struct inode *inode = file->f_mapping->host; 1755 unsigned long limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur; 1756 1757 if (unlikely(*pos < 0)) 1758 return -EINVAL; 1759 1760 if (!isblk) { 1761 /* FIXME: this is for backwards compatibility with 2.4 */ 1762 if (file->f_flags & O_APPEND) 1763 *pos = i_size_read(inode); 1764 1765 if (limit != RLIM_INFINITY) { 1766 if (*pos >= limit) { 1767 send_sig(SIGXFSZ, current, 0); 1768 return -EFBIG; 1769 } 1770 if (*count > limit - (typeof(limit))*pos) { 1771 *count = limit - (typeof(limit))*pos; 1772 } 1773 } 1774 } 1775 1776 /* 1777 * LFS rule 1778 */ 1779 if (unlikely(*pos + *count > MAX_NON_LFS && 1780 !(file->f_flags & O_LARGEFILE))) { 1781 if (*pos >= MAX_NON_LFS) { 1782 send_sig(SIGXFSZ, current, 0); 1783 return -EFBIG; 1784 } 1785 if (*count > MAX_NON_LFS - (unsigned long)*pos) { 1786 *count = MAX_NON_LFS - (unsigned long)*pos; 1787 } 1788 } 1789 1790 /* 1791 * Are we about to exceed the fs block limit ? 1792 * 1793 * If we have written data it becomes a short write. If we have 1794 * exceeded without writing data we send a signal and return EFBIG. 1795 * Linus frestrict idea will clean these up nicely.. 1796 */ 1797 if (likely(!isblk)) { 1798 if (unlikely(*pos >= inode->i_sb->s_maxbytes)) { 1799 if (*count || *pos > inode->i_sb->s_maxbytes) { 1800 send_sig(SIGXFSZ, current, 0); 1801 return -EFBIG; 1802 } 1803 /* zero-length writes at ->s_maxbytes are OK */ 1804 } 1805 1806 if (unlikely(*pos + *count > inode->i_sb->s_maxbytes)) 1807 *count = inode->i_sb->s_maxbytes - *pos; 1808 } else { 1809 loff_t isize; 1810 if (bdev_read_only(I_BDEV(inode))) 1811 return -EPERM; 1812 isize = i_size_read(inode); 1813 if (*pos >= isize) { 1814 if (*count || *pos > isize) 1815 return -ENOSPC; 1816 } 1817 1818 if (*pos + *count > isize) 1819 *count = isize - *pos; 1820 } 1821 return 0; 1822} 1823EXPORT_SYMBOL(generic_write_checks); 1824 1825ssize_t 1826generic_file_direct_write(struct kiocb *iocb, const struct iovec *iov, 1827 unsigned long *nr_segs, loff_t pos, loff_t *ppos, 1828 size_t count, size_t ocount) 1829{ 1830 struct file *file = iocb->ki_filp; 1831 struct address_space *mapping = file->f_mapping; 1832 struct inode *inode = mapping->host; 1833 ssize_t written; 1834 1835 if (count != ocount) 1836 *nr_segs = iov_shorten((struct iovec *)iov, *nr_segs, count); 1837 1838 written = generic_file_direct_IO(WRITE, iocb, iov, pos, *nr_segs); 1839 if (written > 0) { 1840 loff_t end = pos + written; 1841 if (end > i_size_read(inode) && !S_ISBLK(inode->i_mode)) { 1842 i_size_write(inode, end); 1843 mark_inode_dirty(inode); 1844 } 1845 *ppos = end; 1846 } 1847 1848 /* 1849 * Sync the fs metadata but not the minor inode changes and 1850 * of course not the data as we did direct DMA for the IO. 1851 * i_sem is held, which protects generic_osync_inode() from 1852 * livelocking. 1853 */ 1854 if (written >= 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) { 1855 int err = generic_osync_inode(inode, mapping, OSYNC_METADATA); 1856 if (err < 0) 1857 written = err; 1858 } 1859 if (written == count && !is_sync_kiocb(iocb)) 1860 written = -EIOCBQUEUED; 1861 return written; 1862} 1863EXPORT_SYMBOL(generic_file_direct_write); 1864 1865ssize_t 1866generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, 1867 unsigned long nr_segs, loff_t pos, loff_t *ppos, 1868 size_t count, ssize_t written) 1869{ 1870 struct file *file = iocb->ki_filp; 1871 struct address_space * mapping = file->f_mapping; 1872 struct address_space_operations *a_ops = mapping->a_ops; 1873 struct inode *inode = mapping->host; 1874 long status = 0; 1875 struct page *page; 1876 struct page *cached_page = NULL; 1877 size_t bytes; 1878 struct pagevec lru_pvec; 1879 const struct iovec *cur_iov = iov; /* current iovec */ 1880 size_t iov_base = 0; /* offset in the current iovec */ 1881 char __user *buf; 1882 1883 pagevec_init(&lru_pvec, 0); 1884 1885 /* 1886 * handle partial DIO write. Adjust cur_iov if needed. 1887 */ 1888 if (likely(nr_segs == 1)) 1889 buf = iov->iov_base + written; 1890 else { 1891 filemap_set_next_iovec(&cur_iov, &iov_base, written); 1892 buf = cur_iov->iov_base + iov_base; 1893 } 1894 1895 do { 1896 unsigned long index; 1897 unsigned long offset; 1898 unsigned long maxlen; 1899 size_t copied; 1900 1901 offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */ 1902 index = pos >> PAGE_CACHE_SHIFT; 1903 bytes = PAGE_CACHE_SIZE - offset; 1904 if (bytes > count) 1905 bytes = count; 1906 1907 /* 1908 * Bring in the user page that we will copy from _first_. 1909 * Otherwise there's a nasty deadlock on copying from the 1910 * same page as we're writing to, without it being marked 1911 * up-to-date. 1912 */ 1913 maxlen = cur_iov->iov_len - iov_base; 1914 if (maxlen > bytes) 1915 maxlen = bytes; 1916 fault_in_pages_readable(buf, maxlen); 1917 1918 page = __grab_cache_page(mapping,index,&cached_page,&lru_pvec); 1919 if (!page) { 1920 status = -ENOMEM; 1921 break; 1922 } 1923 1924 status = a_ops->prepare_write(file, page, offset, offset+bytes); 1925 if (unlikely(status)) { 1926 loff_t isize = i_size_read(inode); 1927 /* 1928 * prepare_write() may have instantiated a few blocks 1929 * outside i_size. Trim these off again. 1930 */ 1931 unlock_page(page); 1932 page_cache_release(page); 1933 if (pos + bytes > isize) 1934 vmtruncate(inode, isize); 1935 break; 1936 } 1937 if (likely(nr_segs == 1)) 1938 copied = filemap_copy_from_user(page, offset, 1939 buf, bytes); 1940 else 1941 copied = filemap_copy_from_user_iovec(page, offset, 1942 cur_iov, iov_base, bytes); 1943 flush_dcache_page(page); 1944 status = a_ops->commit_write(file, page, offset, offset+bytes); 1945 if (likely(copied > 0)) { 1946 if (!status) 1947 status = copied; 1948 1949 if (status >= 0) { 1950 written += status; 1951 count -= status; 1952 pos += status; 1953 buf += status; 1954 if (unlikely(nr_segs > 1)) { 1955 filemap_set_next_iovec(&cur_iov, 1956 &iov_base, status); 1957 if (count) 1958 buf = cur_iov->iov_base + 1959 iov_base; 1960 } else { 1961 iov_base += status; 1962 } 1963 } 1964 } 1965 if (unlikely(copied != bytes)) 1966 if (status >= 0) 1967 status = -EFAULT; 1968 unlock_page(page); 1969 mark_page_accessed(page); 1970 page_cache_release(page); 1971 if (status < 0) 1972 break; 1973 balance_dirty_pages_ratelimited(mapping); 1974 cond_resched(); 1975 } while (count); 1976 *ppos = pos; 1977 1978 if (cached_page) 1979 page_cache_release(cached_page); 1980 1981 /* 1982 * For now, when the user asks for O_SYNC, we'll actually give O_DSYNC 1983 */ 1984 if (likely(status >= 0)) { 1985 if (unlikely((file->f_flags & O_SYNC) || IS_SYNC(inode))) { 1986 if (!a_ops->writepage || !is_sync_kiocb(iocb)) 1987 status = generic_osync_inode(inode, mapping, 1988 OSYNC_METADATA|OSYNC_DATA); 1989 } 1990 } 1991 1992 /* 1993 * If we get here for O_DIRECT writes then we must have fallen through 1994 * to buffered writes (block instantiation inside i_size). So we sync 1995 * the file data here, to try to honour O_DIRECT expectations. 1996 */ 1997 if (unlikely(file->f_flags & O_DIRECT) && written) 1998 status = filemap_write_and_wait(mapping); 1999 2000 pagevec_lru_add(&lru_pvec); 2001 return written ? written : status; 2002} 2003EXPORT_SYMBOL(generic_file_buffered_write); 2004 2005ssize_t 2006__generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov, 2007 unsigned long nr_segs, loff_t *ppos) 2008{ 2009 struct file *file = iocb->ki_filp; 2010 struct address_space * mapping = file->f_mapping; 2011 size_t ocount; /* original count */ 2012 size_t count; /* after file limit checks */ 2013 struct inode *inode = mapping->host; 2014 unsigned long seg; 2015 loff_t pos; 2016 ssize_t written; 2017 ssize_t err; 2018 2019 ocount = 0; 2020 for (seg = 0; seg < nr_segs; seg++) { 2021 const struct iovec *iv = &iov[seg]; 2022 2023 /* 2024 * If any segment has a negative length, or the cumulative 2025 * length ever wraps negative then return -EINVAL. 2026 */ 2027 ocount += iv->iov_len; 2028 if (unlikely((ssize_t)(ocount|iv->iov_len) < 0)) 2029 return -EINVAL; 2030 if (access_ok(VERIFY_READ, iv->iov_base, iv->iov_len)) 2031 continue; 2032 if (seg == 0) 2033 return -EFAULT; 2034 nr_segs = seg; 2035 ocount -= iv->iov_len; /* This segment is no good */ 2036 break; 2037 } 2038 2039 count = ocount; 2040 pos = *ppos; 2041 2042 vfs_check_frozen(inode->i_sb, SB_FREEZE_WRITE); 2043 2044 /* We can write back this queue in page reclaim */ 2045 current->backing_dev_info = mapping->backing_dev_info; 2046 written = 0; 2047 2048 err = generic_write_checks(file, &pos, &count, S_ISBLK(inode->i_mode)); 2049 if (err) 2050 goto out; 2051 2052 if (count == 0) 2053 goto out; 2054 2055 err = remove_suid(file->f_dentry); 2056 if (err) 2057 goto out; 2058 2059 inode_update_time(inode, 1); 2060 2061 /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */ 2062 if (unlikely(file->f_flags & O_DIRECT)) { 2063 written = generic_file_direct_write(iocb, iov, 2064 &nr_segs, pos, ppos, count, ocount); 2065 if (written < 0 || written == count) 2066 goto out; 2067 /* 2068 * direct-io write to a hole: fall through to buffered I/O 2069 * for completing the rest of the request. 2070 */ 2071 pos += written; 2072 count -= written; 2073 } 2074 2075 written = generic_file_buffered_write(iocb, iov, nr_segs, 2076 pos, ppos, count, written); 2077out: 2078 current->backing_dev_info = NULL; 2079 return written ? written : err; 2080} 2081EXPORT_SYMBOL(generic_file_aio_write_nolock); 2082 2083ssize_t 2084generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov, 2085 unsigned long nr_segs, loff_t *ppos) 2086{ 2087 struct file *file = iocb->ki_filp; 2088 struct address_space *mapping = file->f_mapping; 2089 struct inode *inode = mapping->host; 2090 ssize_t ret; 2091 loff_t pos = *ppos; 2092 2093 ret = __generic_file_aio_write_nolock(iocb, iov, nr_segs, ppos); 2094 2095 if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) { 2096 int err; 2097 2098 err = sync_page_range_nolock(inode, mapping, pos, ret); 2099 if (err < 0) 2100 ret = err; 2101 } 2102 return ret; 2103} 2104 2105ssize_t 2106__generic_file_write_nolock(struct file *file, const struct iovec *iov, 2107 unsigned long nr_segs, loff_t *ppos) 2108{ 2109 struct kiocb kiocb; 2110 ssize_t ret; 2111 2112 init_sync_kiocb(&kiocb, file); 2113 ret = __generic_file_aio_write_nolock(&kiocb, iov, nr_segs, ppos); 2114 if (ret == -EIOCBQUEUED) 2115 ret = wait_on_sync_kiocb(&kiocb); 2116 return ret; 2117} 2118 2119ssize_t 2120generic_file_write_nolock(struct file *file, const struct iovec *iov, 2121 unsigned long nr_segs, loff_t *ppos) 2122{ 2123 struct kiocb kiocb; 2124 ssize_t ret; 2125 2126 init_sync_kiocb(&kiocb, file); 2127 ret = generic_file_aio_write_nolock(&kiocb, iov, nr_segs, ppos); 2128 if (-EIOCBQUEUED == ret) 2129 ret = wait_on_sync_kiocb(&kiocb); 2130 return ret; 2131} 2132EXPORT_SYMBOL(generic_file_write_nolock); 2133 2134ssize_t generic_file_aio_write(struct kiocb *iocb, const char __user *buf, 2135 size_t count, loff_t pos) 2136{ 2137 struct file *file = iocb->ki_filp; 2138 struct address_space *mapping = file->f_mapping; 2139 struct inode *inode = mapping->host; 2140 ssize_t ret; 2141 struct iovec local_iov = { .iov_base = (void __user *)buf, 2142 .iov_len = count }; 2143 2144 BUG_ON(iocb->ki_pos != pos); 2145 2146 down(&inode->i_sem); 2147 ret = __generic_file_aio_write_nolock(iocb, &local_iov, 1, 2148 &iocb->ki_pos); 2149 up(&inode->i_sem); 2150 2151 if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) { 2152 ssize_t err; 2153 2154 err = sync_page_range(inode, mapping, pos, ret); 2155 if (err < 0) 2156 ret = err; 2157 } 2158 return ret; 2159} 2160EXPORT_SYMBOL(generic_file_aio_write); 2161 2162ssize_t generic_file_write(struct file *file, const char __user *buf, 2163 size_t count, loff_t *ppos) 2164{ 2165 struct address_space *mapping = file->f_mapping; 2166 struct inode *inode = mapping->host; 2167 ssize_t ret; 2168 struct iovec local_iov = { .iov_base = (void __user *)buf, 2169 .iov_len = count }; 2170 2171 down(&inode->i_sem); 2172 ret = __generic_file_write_nolock(file, &local_iov, 1, ppos); 2173 up(&inode->i_sem); 2174 2175 if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) { 2176 ssize_t err; 2177 2178 err = sync_page_range(inode, mapping, *ppos - ret, ret); 2179 if (err < 0) 2180 ret = err; 2181 } 2182 return ret; 2183} 2184EXPORT_SYMBOL(generic_file_write); 2185 2186ssize_t generic_file_readv(struct file *filp, const struct iovec *iov, 2187 unsigned long nr_segs, loff_t *ppos) 2188{ 2189 struct kiocb kiocb; 2190 ssize_t ret; 2191 2192 init_sync_kiocb(&kiocb, filp); 2193 ret = __generic_file_aio_read(&kiocb, iov, nr_segs, ppos); 2194 if (-EIOCBQUEUED == ret) 2195 ret = wait_on_sync_kiocb(&kiocb); 2196 return ret; 2197} 2198EXPORT_SYMBOL(generic_file_readv); 2199 2200ssize_t generic_file_writev(struct file *file, const struct iovec *iov, 2201 unsigned long nr_segs, loff_t *ppos) 2202{ 2203 struct address_space *mapping = file->f_mapping; 2204 struct inode *inode = mapping->host; 2205 ssize_t ret; 2206 2207 down(&inode->i_sem); 2208 ret = __generic_file_write_nolock(file, iov, nr_segs, ppos); 2209 up(&inode->i_sem); 2210 2211 if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) { 2212 int err; 2213 2214 err = sync_page_range(inode, mapping, *ppos - ret, ret); 2215 if (err < 0) 2216 ret = err; 2217 } 2218 return ret; 2219} 2220EXPORT_SYMBOL(generic_file_writev); 2221 2222/* 2223 * Called under i_sem for writes to S_ISREG files. Returns -EIO if something 2224 * went wrong during pagecache shootdown. 2225 */ 2226ssize_t 2227generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, 2228 loff_t offset, unsigned long nr_segs) 2229{ 2230 struct file *file = iocb->ki_filp; 2231 struct address_space *mapping = file->f_mapping; 2232 ssize_t retval; 2233 size_t write_len = 0; 2234 2235 /* 2236 * If it's a write, unmap all mmappings of the file up-front. This 2237 * will cause any pte dirty bits to be propagated into the pageframes 2238 * for the subsequent filemap_write_and_wait(). 2239 */ 2240 if (rw == WRITE) { 2241 write_len = iov_length(iov, nr_segs); 2242 if (mapping_mapped(mapping)) 2243 unmap_mapping_range(mapping, offset, write_len, 0); 2244 } 2245 2246 retval = filemap_write_and_wait(mapping); 2247 if (retval == 0) { 2248 retval = mapping->a_ops->direct_IO(rw, iocb, iov, 2249 offset, nr_segs); 2250 if (rw == WRITE && mapping->nrpages) { 2251 pgoff_t end = (offset + write_len - 1) 2252 >> PAGE_CACHE_SHIFT; 2253 int err = invalidate_inode_pages2_range(mapping, 2254 offset >> PAGE_CACHE_SHIFT, end); 2255 if (err) 2256 retval = err; 2257 } 2258 } 2259 return retval; 2260} 2261EXPORT_SYMBOL_GPL(generic_file_direct_IO); 2262