shmem.c revision 69f07ec938712b58755add82dd3d0b35f01317cc
1/* 2 * Resizable virtual memory filesystem for Linux. 3 * 4 * Copyright (C) 2000 Linus Torvalds. 5 * 2000 Transmeta Corp. 6 * 2000-2001 Christoph Rohland 7 * 2000-2001 SAP AG 8 * 2002 Red Hat Inc. 9 * Copyright (C) 2002-2011 Hugh Dickins. 10 * Copyright (C) 2011 Google Inc. 11 * Copyright (C) 2002-2005 VERITAS Software Corporation. 12 * Copyright (C) 2004 Andi Kleen, SuSE Labs 13 * 14 * Extended attribute support for tmpfs: 15 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net> 16 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com> 17 * 18 * tiny-shmem: 19 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com> 20 * 21 * This file is released under the GPL. 22 */ 23 24#include <linux/fs.h> 25#include <linux/init.h> 26#include <linux/vfs.h> 27#include <linux/mount.h> 28#include <linux/pagemap.h> 29#include <linux/file.h> 30#include <linux/mm.h> 31#include <linux/module.h> 32#include <linux/swap.h> 33 34static struct vfsmount *shm_mnt; 35 36#ifdef CONFIG_SHMEM 37/* 38 * This virtual memory filesystem is heavily based on the ramfs. It 39 * extends ramfs by the ability to use swap and honor resource limits 40 * which makes it a completely usable filesystem. 41 */ 42 43#include <linux/xattr.h> 44#include <linux/exportfs.h> 45#include <linux/posix_acl.h> 46#include <linux/generic_acl.h> 47#include <linux/mman.h> 48#include <linux/string.h> 49#include <linux/slab.h> 50#include <linux/backing-dev.h> 51#include <linux/shmem_fs.h> 52#include <linux/writeback.h> 53#include <linux/blkdev.h> 54#include <linux/pagevec.h> 55#include <linux/percpu_counter.h> 56#include <linux/splice.h> 57#include <linux/security.h> 58#include <linux/swapops.h> 59#include <linux/mempolicy.h> 60#include <linux/namei.h> 61#include <linux/ctype.h> 62#include <linux/migrate.h> 63#include <linux/highmem.h> 64#include <linux/seq_file.h> 65#include <linux/magic.h> 66 67#include <asm/uaccess.h> 68#include <asm/pgtable.h> 69 70#define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512) 71#define VM_ACCT(size) (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT) 72 73/* Pretend that each entry is of this size in directory's i_size */ 74#define BOGO_DIRENT_SIZE 20 75 76/* Symlink up to this size is kmalloc'ed instead of using a swappable page */ 77#define SHORT_SYMLINK_LEN 128 78 79struct shmem_xattr { 80 struct list_head list; /* anchored by shmem_inode_info->xattr_list */ 81 char *name; /* xattr name */ 82 size_t size; 83 char value[0]; 84}; 85 86/* Flag allocation requirements to shmem_getpage */ 87enum sgp_type { 88 SGP_READ, /* don't exceed i_size, don't allocate page */ 89 SGP_CACHE, /* don't exceed i_size, may allocate page */ 90 SGP_DIRTY, /* like SGP_CACHE, but set new page dirty */ 91 SGP_WRITE, /* may exceed i_size, may allocate page */ 92}; 93 94#ifdef CONFIG_TMPFS 95static unsigned long shmem_default_max_blocks(void) 96{ 97 return totalram_pages / 2; 98} 99 100static unsigned long shmem_default_max_inodes(void) 101{ 102 return min(totalram_pages - totalhigh_pages, totalram_pages / 2); 103} 104#endif 105 106static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, 107 struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type); 108 109static inline int shmem_getpage(struct inode *inode, pgoff_t index, 110 struct page **pagep, enum sgp_type sgp, int *fault_type) 111{ 112 return shmem_getpage_gfp(inode, index, pagep, sgp, 113 mapping_gfp_mask(inode->i_mapping), fault_type); 114} 115 116static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb) 117{ 118 return sb->s_fs_info; 119} 120 121/* 122 * shmem_file_setup pre-accounts the whole fixed size of a VM object, 123 * for shared memory and for shared anonymous (/dev/zero) mappings 124 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1), 125 * consistent with the pre-accounting of private mappings ... 126 */ 127static inline int shmem_acct_size(unsigned long flags, loff_t size) 128{ 129 return (flags & VM_NORESERVE) ? 130 0 : security_vm_enough_memory_kern(VM_ACCT(size)); 131} 132 133static inline void shmem_unacct_size(unsigned long flags, loff_t size) 134{ 135 if (!(flags & VM_NORESERVE)) 136 vm_unacct_memory(VM_ACCT(size)); 137} 138 139/* 140 * ... whereas tmpfs objects are accounted incrementally as 141 * pages are allocated, in order to allow huge sparse files. 142 * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM, 143 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM. 144 */ 145static inline int shmem_acct_block(unsigned long flags) 146{ 147 return (flags & VM_NORESERVE) ? 148 security_vm_enough_memory_kern(VM_ACCT(PAGE_CACHE_SIZE)) : 0; 149} 150 151static inline void shmem_unacct_blocks(unsigned long flags, long pages) 152{ 153 if (flags & VM_NORESERVE) 154 vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE)); 155} 156 157static const struct super_operations shmem_ops; 158static const struct address_space_operations shmem_aops; 159static const struct file_operations shmem_file_operations; 160static const struct inode_operations shmem_inode_operations; 161static const struct inode_operations shmem_dir_inode_operations; 162static const struct inode_operations shmem_special_inode_operations; 163static const struct vm_operations_struct shmem_vm_ops; 164 165static struct backing_dev_info shmem_backing_dev_info __read_mostly = { 166 .ra_pages = 0, /* No readahead */ 167 .capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED, 168}; 169 170static LIST_HEAD(shmem_swaplist); 171static DEFINE_MUTEX(shmem_swaplist_mutex); 172 173static int shmem_reserve_inode(struct super_block *sb) 174{ 175 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 176 if (sbinfo->max_inodes) { 177 spin_lock(&sbinfo->stat_lock); 178 if (!sbinfo->free_inodes) { 179 spin_unlock(&sbinfo->stat_lock); 180 return -ENOSPC; 181 } 182 sbinfo->free_inodes--; 183 spin_unlock(&sbinfo->stat_lock); 184 } 185 return 0; 186} 187 188static void shmem_free_inode(struct super_block *sb) 189{ 190 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 191 if (sbinfo->max_inodes) { 192 spin_lock(&sbinfo->stat_lock); 193 sbinfo->free_inodes++; 194 spin_unlock(&sbinfo->stat_lock); 195 } 196} 197 198/** 199 * shmem_recalc_inode - recalculate the block usage of an inode 200 * @inode: inode to recalc 201 * 202 * We have to calculate the free blocks since the mm can drop 203 * undirtied hole pages behind our back. 204 * 205 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped 206 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped) 207 * 208 * It has to be called with the spinlock held. 209 */ 210static void shmem_recalc_inode(struct inode *inode) 211{ 212 struct shmem_inode_info *info = SHMEM_I(inode); 213 long freed; 214 215 freed = info->alloced - info->swapped - inode->i_mapping->nrpages; 216 if (freed > 0) { 217 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 218 if (sbinfo->max_blocks) 219 percpu_counter_add(&sbinfo->used_blocks, -freed); 220 info->alloced -= freed; 221 inode->i_blocks -= freed * BLOCKS_PER_PAGE; 222 shmem_unacct_blocks(info->flags, freed); 223 } 224} 225 226/* 227 * Replace item expected in radix tree by a new item, while holding tree lock. 228 */ 229static int shmem_radix_tree_replace(struct address_space *mapping, 230 pgoff_t index, void *expected, void *replacement) 231{ 232 void **pslot; 233 void *item = NULL; 234 235 VM_BUG_ON(!expected); 236 pslot = radix_tree_lookup_slot(&mapping->page_tree, index); 237 if (pslot) 238 item = radix_tree_deref_slot_protected(pslot, 239 &mapping->tree_lock); 240 if (item != expected) 241 return -ENOENT; 242 if (replacement) 243 radix_tree_replace_slot(pslot, replacement); 244 else 245 radix_tree_delete(&mapping->page_tree, index); 246 return 0; 247} 248 249/* 250 * Like add_to_page_cache_locked, but error if expected item has gone. 251 */ 252static int shmem_add_to_page_cache(struct page *page, 253 struct address_space *mapping, 254 pgoff_t index, gfp_t gfp, void *expected) 255{ 256 int error = 0; 257 258 VM_BUG_ON(!PageLocked(page)); 259 VM_BUG_ON(!PageSwapBacked(page)); 260 261 if (!expected) 262 error = radix_tree_preload(gfp & GFP_RECLAIM_MASK); 263 if (!error) { 264 page_cache_get(page); 265 page->mapping = mapping; 266 page->index = index; 267 268 spin_lock_irq(&mapping->tree_lock); 269 if (!expected) 270 error = radix_tree_insert(&mapping->page_tree, 271 index, page); 272 else 273 error = shmem_radix_tree_replace(mapping, index, 274 expected, page); 275 if (!error) { 276 mapping->nrpages++; 277 __inc_zone_page_state(page, NR_FILE_PAGES); 278 __inc_zone_page_state(page, NR_SHMEM); 279 spin_unlock_irq(&mapping->tree_lock); 280 } else { 281 page->mapping = NULL; 282 spin_unlock_irq(&mapping->tree_lock); 283 page_cache_release(page); 284 } 285 if (!expected) 286 radix_tree_preload_end(); 287 } 288 if (error) 289 mem_cgroup_uncharge_cache_page(page); 290 return error; 291} 292 293/* 294 * Like delete_from_page_cache, but substitutes swap for page. 295 */ 296static void shmem_delete_from_page_cache(struct page *page, void *radswap) 297{ 298 struct address_space *mapping = page->mapping; 299 int error; 300 301 spin_lock_irq(&mapping->tree_lock); 302 error = shmem_radix_tree_replace(mapping, page->index, page, radswap); 303 page->mapping = NULL; 304 mapping->nrpages--; 305 __dec_zone_page_state(page, NR_FILE_PAGES); 306 __dec_zone_page_state(page, NR_SHMEM); 307 spin_unlock_irq(&mapping->tree_lock); 308 page_cache_release(page); 309 BUG_ON(error); 310} 311 312/* 313 * Like find_get_pages, but collecting swap entries as well as pages. 314 */ 315static unsigned shmem_find_get_pages_and_swap(struct address_space *mapping, 316 pgoff_t start, unsigned int nr_pages, 317 struct page **pages, pgoff_t *indices) 318{ 319 unsigned int i; 320 unsigned int ret; 321 unsigned int nr_found; 322 323 rcu_read_lock(); 324restart: 325 nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree, 326 (void ***)pages, indices, start, nr_pages); 327 ret = 0; 328 for (i = 0; i < nr_found; i++) { 329 struct page *page; 330repeat: 331 page = radix_tree_deref_slot((void **)pages[i]); 332 if (unlikely(!page)) 333 continue; 334 if (radix_tree_exception(page)) { 335 if (radix_tree_exceptional_entry(page)) 336 goto export; 337 /* radix_tree_deref_retry(page) */ 338 goto restart; 339 } 340 if (!page_cache_get_speculative(page)) 341 goto repeat; 342 343 /* Has the page moved? */ 344 if (unlikely(page != *((void **)pages[i]))) { 345 page_cache_release(page); 346 goto repeat; 347 } 348export: 349 indices[ret] = indices[i]; 350 pages[ret] = page; 351 ret++; 352 } 353 if (unlikely(!ret && nr_found)) 354 goto restart; 355 rcu_read_unlock(); 356 return ret; 357} 358 359/* 360 * Lockless lookup of swap entry in radix tree, avoiding refcount on pages. 361 */ 362static pgoff_t shmem_find_swap(struct address_space *mapping, void *radswap) 363{ 364 void **slots[PAGEVEC_SIZE]; 365 pgoff_t indices[PAGEVEC_SIZE]; 366 unsigned int nr_found; 367 368restart: 369 nr_found = 1; 370 indices[0] = -1; 371 while (nr_found) { 372 pgoff_t index = indices[nr_found - 1] + 1; 373 unsigned int i; 374 375 rcu_read_lock(); 376 nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree, 377 slots, indices, index, PAGEVEC_SIZE); 378 for (i = 0; i < nr_found; i++) { 379 void *item = radix_tree_deref_slot(slots[i]); 380 if (radix_tree_deref_retry(item)) { 381 rcu_read_unlock(); 382 goto restart; 383 } 384 if (item == radswap) { 385 rcu_read_unlock(); 386 return indices[i]; 387 } 388 } 389 rcu_read_unlock(); 390 cond_resched(); 391 } 392 return -1; 393} 394 395/* 396 * Remove swap entry from radix tree, free the swap and its page cache. 397 */ 398static int shmem_free_swap(struct address_space *mapping, 399 pgoff_t index, void *radswap) 400{ 401 int error; 402 403 spin_lock_irq(&mapping->tree_lock); 404 error = shmem_radix_tree_replace(mapping, index, radswap, NULL); 405 spin_unlock_irq(&mapping->tree_lock); 406 if (!error) 407 free_swap_and_cache(radix_to_swp_entry(radswap)); 408 return error; 409} 410 411/* 412 * Pagevec may contain swap entries, so shuffle up pages before releasing. 413 */ 414static void shmem_pagevec_release(struct pagevec *pvec) 415{ 416 int i, j; 417 418 for (i = 0, j = 0; i < pagevec_count(pvec); i++) { 419 struct page *page = pvec->pages[i]; 420 if (!radix_tree_exceptional_entry(page)) 421 pvec->pages[j++] = page; 422 } 423 pvec->nr = j; 424 pagevec_release(pvec); 425} 426 427/* 428 * Remove range of pages and swap entries from radix tree, and free them. 429 */ 430void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 431{ 432 struct address_space *mapping = inode->i_mapping; 433 struct shmem_inode_info *info = SHMEM_I(inode); 434 pgoff_t start = (lstart + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 435 unsigned partial = lstart & (PAGE_CACHE_SIZE - 1); 436 pgoff_t end = (lend >> PAGE_CACHE_SHIFT); 437 struct pagevec pvec; 438 pgoff_t indices[PAGEVEC_SIZE]; 439 long nr_swaps_freed = 0; 440 pgoff_t index; 441 int i; 442 443 BUG_ON((lend & (PAGE_CACHE_SIZE - 1)) != (PAGE_CACHE_SIZE - 1)); 444 445 pagevec_init(&pvec, 0); 446 index = start; 447 while (index <= end) { 448 pvec.nr = shmem_find_get_pages_and_swap(mapping, index, 449 min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1, 450 pvec.pages, indices); 451 if (!pvec.nr) 452 break; 453 mem_cgroup_uncharge_start(); 454 for (i = 0; i < pagevec_count(&pvec); i++) { 455 struct page *page = pvec.pages[i]; 456 457 index = indices[i]; 458 if (index > end) 459 break; 460 461 if (radix_tree_exceptional_entry(page)) { 462 nr_swaps_freed += !shmem_free_swap(mapping, 463 index, page); 464 continue; 465 } 466 467 if (!trylock_page(page)) 468 continue; 469 if (page->mapping == mapping) { 470 VM_BUG_ON(PageWriteback(page)); 471 truncate_inode_page(mapping, page); 472 } 473 unlock_page(page); 474 } 475 shmem_pagevec_release(&pvec); 476 mem_cgroup_uncharge_end(); 477 cond_resched(); 478 index++; 479 } 480 481 if (partial) { 482 struct page *page = NULL; 483 shmem_getpage(inode, start - 1, &page, SGP_READ, NULL); 484 if (page) { 485 zero_user_segment(page, partial, PAGE_CACHE_SIZE); 486 set_page_dirty(page); 487 unlock_page(page); 488 page_cache_release(page); 489 } 490 } 491 492 index = start; 493 for ( ; ; ) { 494 cond_resched(); 495 pvec.nr = shmem_find_get_pages_and_swap(mapping, index, 496 min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1, 497 pvec.pages, indices); 498 if (!pvec.nr) { 499 if (index == start) 500 break; 501 index = start; 502 continue; 503 } 504 if (index == start && indices[0] > end) { 505 shmem_pagevec_release(&pvec); 506 break; 507 } 508 mem_cgroup_uncharge_start(); 509 for (i = 0; i < pagevec_count(&pvec); i++) { 510 struct page *page = pvec.pages[i]; 511 512 index = indices[i]; 513 if (index > end) 514 break; 515 516 if (radix_tree_exceptional_entry(page)) { 517 nr_swaps_freed += !shmem_free_swap(mapping, 518 index, page); 519 continue; 520 } 521 522 lock_page(page); 523 if (page->mapping == mapping) { 524 VM_BUG_ON(PageWriteback(page)); 525 truncate_inode_page(mapping, page); 526 } 527 unlock_page(page); 528 } 529 shmem_pagevec_release(&pvec); 530 mem_cgroup_uncharge_end(); 531 index++; 532 } 533 534 spin_lock(&info->lock); 535 info->swapped -= nr_swaps_freed; 536 shmem_recalc_inode(inode); 537 spin_unlock(&info->lock); 538 539 inode->i_ctime = inode->i_mtime = CURRENT_TIME; 540} 541EXPORT_SYMBOL_GPL(shmem_truncate_range); 542 543static int shmem_setattr(struct dentry *dentry, struct iattr *attr) 544{ 545 struct inode *inode = dentry->d_inode; 546 int error; 547 548 error = inode_change_ok(inode, attr); 549 if (error) 550 return error; 551 552 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 553 loff_t oldsize = inode->i_size; 554 loff_t newsize = attr->ia_size; 555 556 if (newsize != oldsize) { 557 i_size_write(inode, newsize); 558 inode->i_ctime = inode->i_mtime = CURRENT_TIME; 559 } 560 if (newsize < oldsize) { 561 loff_t holebegin = round_up(newsize, PAGE_SIZE); 562 unmap_mapping_range(inode->i_mapping, holebegin, 0, 1); 563 shmem_truncate_range(inode, newsize, (loff_t)-1); 564 /* unmap again to remove racily COWed private pages */ 565 unmap_mapping_range(inode->i_mapping, holebegin, 0, 1); 566 } 567 } 568 569 setattr_copy(inode, attr); 570#ifdef CONFIG_TMPFS_POSIX_ACL 571 if (attr->ia_valid & ATTR_MODE) 572 error = generic_acl_chmod(inode); 573#endif 574 return error; 575} 576 577static void shmem_evict_inode(struct inode *inode) 578{ 579 struct shmem_inode_info *info = SHMEM_I(inode); 580 struct shmem_xattr *xattr, *nxattr; 581 582 if (inode->i_mapping->a_ops == &shmem_aops) { 583 shmem_unacct_size(info->flags, inode->i_size); 584 inode->i_size = 0; 585 shmem_truncate_range(inode, 0, (loff_t)-1); 586 if (!list_empty(&info->swaplist)) { 587 mutex_lock(&shmem_swaplist_mutex); 588 list_del_init(&info->swaplist); 589 mutex_unlock(&shmem_swaplist_mutex); 590 } 591 } else 592 kfree(info->symlink); 593 594 list_for_each_entry_safe(xattr, nxattr, &info->xattr_list, list) { 595 kfree(xattr->name); 596 kfree(xattr); 597 } 598 BUG_ON(inode->i_blocks); 599 shmem_free_inode(inode->i_sb); 600 end_writeback(inode); 601} 602 603/* 604 * If swap found in inode, free it and move page from swapcache to filecache. 605 */ 606static int shmem_unuse_inode(struct shmem_inode_info *info, 607 swp_entry_t swap, struct page *page) 608{ 609 struct address_space *mapping = info->vfs_inode.i_mapping; 610 void *radswap; 611 pgoff_t index; 612 int error; 613 614 radswap = swp_to_radix_entry(swap); 615 index = shmem_find_swap(mapping, radswap); 616 if (index == -1) 617 return 0; 618 619 /* 620 * Move _head_ to start search for next from here. 621 * But be careful: shmem_evict_inode checks list_empty without taking 622 * mutex, and there's an instant in list_move_tail when info->swaplist 623 * would appear empty, if it were the only one on shmem_swaplist. 624 */ 625 if (shmem_swaplist.next != &info->swaplist) 626 list_move_tail(&shmem_swaplist, &info->swaplist); 627 628 /* 629 * We rely on shmem_swaplist_mutex, not only to protect the swaplist, 630 * but also to hold up shmem_evict_inode(): so inode cannot be freed 631 * beneath us (pagelock doesn't help until the page is in pagecache). 632 */ 633 error = shmem_add_to_page_cache(page, mapping, index, 634 GFP_NOWAIT, radswap); 635 /* which does mem_cgroup_uncharge_cache_page on error */ 636 637 if (error != -ENOMEM) { 638 /* 639 * Truncation and eviction use free_swap_and_cache(), which 640 * only does trylock page: if we raced, best clean up here. 641 */ 642 delete_from_swap_cache(page); 643 set_page_dirty(page); 644 if (!error) { 645 spin_lock(&info->lock); 646 info->swapped--; 647 spin_unlock(&info->lock); 648 swap_free(swap); 649 } 650 error = 1; /* not an error, but entry was found */ 651 } 652 return error; 653} 654 655/* 656 * Search through swapped inodes to find and replace swap by page. 657 */ 658int shmem_unuse(swp_entry_t swap, struct page *page) 659{ 660 struct list_head *this, *next; 661 struct shmem_inode_info *info; 662 int found = 0; 663 int error; 664 665 /* 666 * Charge page using GFP_KERNEL while we can wait, before taking 667 * the shmem_swaplist_mutex which might hold up shmem_writepage(). 668 * Charged back to the user (not to caller) when swap account is used. 669 */ 670 error = mem_cgroup_cache_charge(page, current->mm, GFP_KERNEL); 671 if (error) 672 goto out; 673 /* No radix_tree_preload: swap entry keeps a place for page in tree */ 674 675 mutex_lock(&shmem_swaplist_mutex); 676 list_for_each_safe(this, next, &shmem_swaplist) { 677 info = list_entry(this, struct shmem_inode_info, swaplist); 678 if (info->swapped) 679 found = shmem_unuse_inode(info, swap, page); 680 else 681 list_del_init(&info->swaplist); 682 cond_resched(); 683 if (found) 684 break; 685 } 686 mutex_unlock(&shmem_swaplist_mutex); 687 688 if (!found) 689 mem_cgroup_uncharge_cache_page(page); 690 if (found < 0) 691 error = found; 692out: 693 unlock_page(page); 694 page_cache_release(page); 695 return error; 696} 697 698/* 699 * Move the page from the page cache to the swap cache. 700 */ 701static int shmem_writepage(struct page *page, struct writeback_control *wbc) 702{ 703 struct shmem_inode_info *info; 704 struct address_space *mapping; 705 struct inode *inode; 706 swp_entry_t swap; 707 pgoff_t index; 708 709 BUG_ON(!PageLocked(page)); 710 mapping = page->mapping; 711 index = page->index; 712 inode = mapping->host; 713 info = SHMEM_I(inode); 714 if (info->flags & VM_LOCKED) 715 goto redirty; 716 if (!total_swap_pages) 717 goto redirty; 718 719 /* 720 * shmem_backing_dev_info's capabilities prevent regular writeback or 721 * sync from ever calling shmem_writepage; but a stacking filesystem 722 * might use ->writepage of its underlying filesystem, in which case 723 * tmpfs should write out to swap only in response to memory pressure, 724 * and not for the writeback threads or sync. 725 */ 726 if (!wbc->for_reclaim) { 727 WARN_ON_ONCE(1); /* Still happens? Tell us about it! */ 728 goto redirty; 729 } 730 swap = get_swap_page(); 731 if (!swap.val) 732 goto redirty; 733 734 /* 735 * Add inode to shmem_unuse()'s list of swapped-out inodes, 736 * if it's not already there. Do it now before the page is 737 * moved to swap cache, when its pagelock no longer protects 738 * the inode from eviction. But don't unlock the mutex until 739 * we've incremented swapped, because shmem_unuse_inode() will 740 * prune a !swapped inode from the swaplist under this mutex. 741 */ 742 mutex_lock(&shmem_swaplist_mutex); 743 if (list_empty(&info->swaplist)) 744 list_add_tail(&info->swaplist, &shmem_swaplist); 745 746 if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) { 747 swap_shmem_alloc(swap); 748 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap)); 749 750 spin_lock(&info->lock); 751 info->swapped++; 752 shmem_recalc_inode(inode); 753 spin_unlock(&info->lock); 754 755 mutex_unlock(&shmem_swaplist_mutex); 756 BUG_ON(page_mapped(page)); 757 swap_writepage(page, wbc); 758 return 0; 759 } 760 761 mutex_unlock(&shmem_swaplist_mutex); 762 swapcache_free(swap, NULL); 763redirty: 764 set_page_dirty(page); 765 if (wbc->for_reclaim) 766 return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */ 767 unlock_page(page); 768 return 0; 769} 770 771#ifdef CONFIG_NUMA 772#ifdef CONFIG_TMPFS 773static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 774{ 775 char buffer[64]; 776 777 if (!mpol || mpol->mode == MPOL_DEFAULT) 778 return; /* show nothing */ 779 780 mpol_to_str(buffer, sizeof(buffer), mpol, 1); 781 782 seq_printf(seq, ",mpol=%s", buffer); 783} 784 785static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 786{ 787 struct mempolicy *mpol = NULL; 788 if (sbinfo->mpol) { 789 spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */ 790 mpol = sbinfo->mpol; 791 mpol_get(mpol); 792 spin_unlock(&sbinfo->stat_lock); 793 } 794 return mpol; 795} 796#endif /* CONFIG_TMPFS */ 797 798static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp, 799 struct shmem_inode_info *info, pgoff_t index) 800{ 801 struct mempolicy mpol, *spol; 802 struct vm_area_struct pvma; 803 804 spol = mpol_cond_copy(&mpol, 805 mpol_shared_policy_lookup(&info->policy, index)); 806 807 /* Create a pseudo vma that just contains the policy */ 808 pvma.vm_start = 0; 809 pvma.vm_pgoff = index; 810 pvma.vm_ops = NULL; 811 pvma.vm_policy = spol; 812 return swapin_readahead(swap, gfp, &pvma, 0); 813} 814 815static struct page *shmem_alloc_page(gfp_t gfp, 816 struct shmem_inode_info *info, pgoff_t index) 817{ 818 struct vm_area_struct pvma; 819 820 /* Create a pseudo vma that just contains the policy */ 821 pvma.vm_start = 0; 822 pvma.vm_pgoff = index; 823 pvma.vm_ops = NULL; 824 pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, index); 825 826 /* 827 * alloc_page_vma() will drop the shared policy reference 828 */ 829 return alloc_page_vma(gfp, &pvma, 0); 830} 831#else /* !CONFIG_NUMA */ 832#ifdef CONFIG_TMPFS 833static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 834{ 835} 836#endif /* CONFIG_TMPFS */ 837 838static inline struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp, 839 struct shmem_inode_info *info, pgoff_t index) 840{ 841 return swapin_readahead(swap, gfp, NULL, 0); 842} 843 844static inline struct page *shmem_alloc_page(gfp_t gfp, 845 struct shmem_inode_info *info, pgoff_t index) 846{ 847 return alloc_page(gfp); 848} 849#endif /* CONFIG_NUMA */ 850 851#if !defined(CONFIG_NUMA) || !defined(CONFIG_TMPFS) 852static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 853{ 854 return NULL; 855} 856#endif 857 858/* 859 * shmem_getpage_gfp - find page in cache, or get from swap, or allocate 860 * 861 * If we allocate a new one we do not mark it dirty. That's up to the 862 * vm. If we swap it in we mark it dirty since we also free the swap 863 * entry since a page cannot live in both the swap and page cache 864 */ 865static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, 866 struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type) 867{ 868 struct address_space *mapping = inode->i_mapping; 869 struct shmem_inode_info *info; 870 struct shmem_sb_info *sbinfo; 871 struct page *page; 872 swp_entry_t swap; 873 int error; 874 int once = 0; 875 876 if (index > (MAX_LFS_FILESIZE >> PAGE_CACHE_SHIFT)) 877 return -EFBIG; 878repeat: 879 swap.val = 0; 880 page = find_lock_page(mapping, index); 881 if (radix_tree_exceptional_entry(page)) { 882 swap = radix_to_swp_entry(page); 883 page = NULL; 884 } 885 886 if (sgp != SGP_WRITE && 887 ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) { 888 error = -EINVAL; 889 goto failed; 890 } 891 892 if (page || (sgp == SGP_READ && !swap.val)) { 893 /* 894 * Once we can get the page lock, it must be uptodate: 895 * if there were an error in reading back from swap, 896 * the page would not be inserted into the filecache. 897 */ 898 BUG_ON(page && !PageUptodate(page)); 899 *pagep = page; 900 return 0; 901 } 902 903 /* 904 * Fast cache lookup did not find it: 905 * bring it back from swap or allocate. 906 */ 907 info = SHMEM_I(inode); 908 sbinfo = SHMEM_SB(inode->i_sb); 909 910 if (swap.val) { 911 /* Look it up and read it in.. */ 912 page = lookup_swap_cache(swap); 913 if (!page) { 914 /* here we actually do the io */ 915 if (fault_type) 916 *fault_type |= VM_FAULT_MAJOR; 917 page = shmem_swapin(swap, gfp, info, index); 918 if (!page) { 919 error = -ENOMEM; 920 goto failed; 921 } 922 } 923 924 /* We have to do this with page locked to prevent races */ 925 lock_page(page); 926 if (!PageUptodate(page)) { 927 error = -EIO; 928 goto failed; 929 } 930 wait_on_page_writeback(page); 931 932 /* Someone may have already done it for us */ 933 if (page->mapping) { 934 if (page->mapping == mapping && 935 page->index == index) 936 goto done; 937 error = -EEXIST; 938 goto failed; 939 } 940 941 error = mem_cgroup_cache_charge(page, current->mm, 942 gfp & GFP_RECLAIM_MASK); 943 if (!error) 944 error = shmem_add_to_page_cache(page, mapping, index, 945 gfp, swp_to_radix_entry(swap)); 946 if (error) 947 goto failed; 948 949 spin_lock(&info->lock); 950 info->swapped--; 951 shmem_recalc_inode(inode); 952 spin_unlock(&info->lock); 953 954 delete_from_swap_cache(page); 955 set_page_dirty(page); 956 swap_free(swap); 957 958 } else { 959 if (shmem_acct_block(info->flags)) { 960 error = -ENOSPC; 961 goto failed; 962 } 963 if (sbinfo->max_blocks) { 964 if (percpu_counter_compare(&sbinfo->used_blocks, 965 sbinfo->max_blocks) >= 0) { 966 error = -ENOSPC; 967 goto unacct; 968 } 969 percpu_counter_inc(&sbinfo->used_blocks); 970 } 971 972 page = shmem_alloc_page(gfp, info, index); 973 if (!page) { 974 error = -ENOMEM; 975 goto decused; 976 } 977 978 SetPageSwapBacked(page); 979 __set_page_locked(page); 980 error = mem_cgroup_cache_charge(page, current->mm, 981 gfp & GFP_RECLAIM_MASK); 982 if (!error) 983 error = shmem_add_to_page_cache(page, mapping, index, 984 gfp, NULL); 985 if (error) 986 goto decused; 987 lru_cache_add_anon(page); 988 989 spin_lock(&info->lock); 990 info->alloced++; 991 inode->i_blocks += BLOCKS_PER_PAGE; 992 shmem_recalc_inode(inode); 993 spin_unlock(&info->lock); 994 995 clear_highpage(page); 996 flush_dcache_page(page); 997 SetPageUptodate(page); 998 if (sgp == SGP_DIRTY) 999 set_page_dirty(page); 1000 } 1001done: 1002 /* Perhaps the file has been truncated since we checked */ 1003 if (sgp != SGP_WRITE && 1004 ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) { 1005 error = -EINVAL; 1006 goto trunc; 1007 } 1008 *pagep = page; 1009 return 0; 1010 1011 /* 1012 * Error recovery. 1013 */ 1014trunc: 1015 ClearPageDirty(page); 1016 delete_from_page_cache(page); 1017 spin_lock(&info->lock); 1018 info->alloced--; 1019 inode->i_blocks -= BLOCKS_PER_PAGE; 1020 spin_unlock(&info->lock); 1021decused: 1022 if (sbinfo->max_blocks) 1023 percpu_counter_add(&sbinfo->used_blocks, -1); 1024unacct: 1025 shmem_unacct_blocks(info->flags, 1); 1026failed: 1027 if (swap.val && error != -EINVAL) { 1028 struct page *test = find_get_page(mapping, index); 1029 if (test && !radix_tree_exceptional_entry(test)) 1030 page_cache_release(test); 1031 /* Have another try if the entry has changed */ 1032 if (test != swp_to_radix_entry(swap)) 1033 error = -EEXIST; 1034 } 1035 if (page) { 1036 unlock_page(page); 1037 page_cache_release(page); 1038 } 1039 if (error == -ENOSPC && !once++) { 1040 info = SHMEM_I(inode); 1041 spin_lock(&info->lock); 1042 shmem_recalc_inode(inode); 1043 spin_unlock(&info->lock); 1044 goto repeat; 1045 } 1046 if (error == -EEXIST) 1047 goto repeat; 1048 return error; 1049} 1050 1051static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1052{ 1053 struct inode *inode = vma->vm_file->f_path.dentry->d_inode; 1054 int error; 1055 int ret = VM_FAULT_LOCKED; 1056 1057 error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret); 1058 if (error) 1059 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS); 1060 1061 if (ret & VM_FAULT_MAJOR) { 1062 count_vm_event(PGMAJFAULT); 1063 mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT); 1064 } 1065 return ret; 1066} 1067 1068#ifdef CONFIG_NUMA 1069static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol) 1070{ 1071 struct inode *inode = vma->vm_file->f_path.dentry->d_inode; 1072 return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol); 1073} 1074 1075static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, 1076 unsigned long addr) 1077{ 1078 struct inode *inode = vma->vm_file->f_path.dentry->d_inode; 1079 pgoff_t index; 1080 1081 index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 1082 return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); 1083} 1084#endif 1085 1086int shmem_lock(struct file *file, int lock, struct user_struct *user) 1087{ 1088 struct inode *inode = file->f_path.dentry->d_inode; 1089 struct shmem_inode_info *info = SHMEM_I(inode); 1090 int retval = -ENOMEM; 1091 1092 spin_lock(&info->lock); 1093 if (lock && !(info->flags & VM_LOCKED)) { 1094 if (!user_shm_lock(inode->i_size, user)) 1095 goto out_nomem; 1096 info->flags |= VM_LOCKED; 1097 mapping_set_unevictable(file->f_mapping); 1098 } 1099 if (!lock && (info->flags & VM_LOCKED) && user) { 1100 user_shm_unlock(inode->i_size, user); 1101 info->flags &= ~VM_LOCKED; 1102 mapping_clear_unevictable(file->f_mapping); 1103 scan_mapping_unevictable_pages(file->f_mapping); 1104 } 1105 retval = 0; 1106 1107out_nomem: 1108 spin_unlock(&info->lock); 1109 return retval; 1110} 1111 1112static int shmem_mmap(struct file *file, struct vm_area_struct *vma) 1113{ 1114 file_accessed(file); 1115 vma->vm_ops = &shmem_vm_ops; 1116 vma->vm_flags |= VM_CAN_NONLINEAR; 1117 return 0; 1118} 1119 1120static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir, 1121 int mode, dev_t dev, unsigned long flags) 1122{ 1123 struct inode *inode; 1124 struct shmem_inode_info *info; 1125 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 1126 1127 if (shmem_reserve_inode(sb)) 1128 return NULL; 1129 1130 inode = new_inode(sb); 1131 if (inode) { 1132 inode->i_ino = get_next_ino(); 1133 inode_init_owner(inode, dir, mode); 1134 inode->i_blocks = 0; 1135 inode->i_mapping->backing_dev_info = &shmem_backing_dev_info; 1136 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME; 1137 inode->i_generation = get_seconds(); 1138 info = SHMEM_I(inode); 1139 memset(info, 0, (char *)inode - (char *)info); 1140 spin_lock_init(&info->lock); 1141 info->flags = flags & VM_NORESERVE; 1142 INIT_LIST_HEAD(&info->swaplist); 1143 INIT_LIST_HEAD(&info->xattr_list); 1144 cache_no_acl(inode); 1145 1146 switch (mode & S_IFMT) { 1147 default: 1148 inode->i_op = &shmem_special_inode_operations; 1149 init_special_inode(inode, mode, dev); 1150 break; 1151 case S_IFREG: 1152 inode->i_mapping->a_ops = &shmem_aops; 1153 inode->i_op = &shmem_inode_operations; 1154 inode->i_fop = &shmem_file_operations; 1155 mpol_shared_policy_init(&info->policy, 1156 shmem_get_sbmpol(sbinfo)); 1157 break; 1158 case S_IFDIR: 1159 inc_nlink(inode); 1160 /* Some things misbehave if size == 0 on a directory */ 1161 inode->i_size = 2 * BOGO_DIRENT_SIZE; 1162 inode->i_op = &shmem_dir_inode_operations; 1163 inode->i_fop = &simple_dir_operations; 1164 break; 1165 case S_IFLNK: 1166 /* 1167 * Must not load anything in the rbtree, 1168 * mpol_free_shared_policy will not be called. 1169 */ 1170 mpol_shared_policy_init(&info->policy, NULL); 1171 break; 1172 } 1173 } else 1174 shmem_free_inode(sb); 1175 return inode; 1176} 1177 1178#ifdef CONFIG_TMPFS 1179static const struct inode_operations shmem_symlink_inode_operations; 1180static const struct inode_operations shmem_short_symlink_operations; 1181 1182static int 1183shmem_write_begin(struct file *file, struct address_space *mapping, 1184 loff_t pos, unsigned len, unsigned flags, 1185 struct page **pagep, void **fsdata) 1186{ 1187 struct inode *inode = mapping->host; 1188 pgoff_t index = pos >> PAGE_CACHE_SHIFT; 1189 return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL); 1190} 1191 1192static int 1193shmem_write_end(struct file *file, struct address_space *mapping, 1194 loff_t pos, unsigned len, unsigned copied, 1195 struct page *page, void *fsdata) 1196{ 1197 struct inode *inode = mapping->host; 1198 1199 if (pos + copied > inode->i_size) 1200 i_size_write(inode, pos + copied); 1201 1202 set_page_dirty(page); 1203 unlock_page(page); 1204 page_cache_release(page); 1205 1206 return copied; 1207} 1208 1209static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor) 1210{ 1211 struct inode *inode = filp->f_path.dentry->d_inode; 1212 struct address_space *mapping = inode->i_mapping; 1213 pgoff_t index; 1214 unsigned long offset; 1215 enum sgp_type sgp = SGP_READ; 1216 1217 /* 1218 * Might this read be for a stacking filesystem? Then when reading 1219 * holes of a sparse file, we actually need to allocate those pages, 1220 * and even mark them dirty, so it cannot exceed the max_blocks limit. 1221 */ 1222 if (segment_eq(get_fs(), KERNEL_DS)) 1223 sgp = SGP_DIRTY; 1224 1225 index = *ppos >> PAGE_CACHE_SHIFT; 1226 offset = *ppos & ~PAGE_CACHE_MASK; 1227 1228 for (;;) { 1229 struct page *page = NULL; 1230 pgoff_t end_index; 1231 unsigned long nr, ret; 1232 loff_t i_size = i_size_read(inode); 1233 1234 end_index = i_size >> PAGE_CACHE_SHIFT; 1235 if (index > end_index) 1236 break; 1237 if (index == end_index) { 1238 nr = i_size & ~PAGE_CACHE_MASK; 1239 if (nr <= offset) 1240 break; 1241 } 1242 1243 desc->error = shmem_getpage(inode, index, &page, sgp, NULL); 1244 if (desc->error) { 1245 if (desc->error == -EINVAL) 1246 desc->error = 0; 1247 break; 1248 } 1249 if (page) 1250 unlock_page(page); 1251 1252 /* 1253 * We must evaluate after, since reads (unlike writes) 1254 * are called without i_mutex protection against truncate 1255 */ 1256 nr = PAGE_CACHE_SIZE; 1257 i_size = i_size_read(inode); 1258 end_index = i_size >> PAGE_CACHE_SHIFT; 1259 if (index == end_index) { 1260 nr = i_size & ~PAGE_CACHE_MASK; 1261 if (nr <= offset) { 1262 if (page) 1263 page_cache_release(page); 1264 break; 1265 } 1266 } 1267 nr -= offset; 1268 1269 if (page) { 1270 /* 1271 * If users can be writing to this page using arbitrary 1272 * virtual addresses, take care about potential aliasing 1273 * before reading the page on the kernel side. 1274 */ 1275 if (mapping_writably_mapped(mapping)) 1276 flush_dcache_page(page); 1277 /* 1278 * Mark the page accessed if we read the beginning. 1279 */ 1280 if (!offset) 1281 mark_page_accessed(page); 1282 } else { 1283 page = ZERO_PAGE(0); 1284 page_cache_get(page); 1285 } 1286 1287 /* 1288 * Ok, we have the page, and it's up-to-date, so 1289 * now we can copy it to user space... 1290 * 1291 * The actor routine returns how many bytes were actually used.. 1292 * NOTE! This may not be the same as how much of a user buffer 1293 * we filled up (we may be padding etc), so we can only update 1294 * "pos" here (the actor routine has to update the user buffer 1295 * pointers and the remaining count). 1296 */ 1297 ret = actor(desc, page, offset, nr); 1298 offset += ret; 1299 index += offset >> PAGE_CACHE_SHIFT; 1300 offset &= ~PAGE_CACHE_MASK; 1301 1302 page_cache_release(page); 1303 if (ret != nr || !desc->count) 1304 break; 1305 1306 cond_resched(); 1307 } 1308 1309 *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset; 1310 file_accessed(filp); 1311} 1312 1313static ssize_t shmem_file_aio_read(struct kiocb *iocb, 1314 const struct iovec *iov, unsigned long nr_segs, loff_t pos) 1315{ 1316 struct file *filp = iocb->ki_filp; 1317 ssize_t retval; 1318 unsigned long seg; 1319 size_t count; 1320 loff_t *ppos = &iocb->ki_pos; 1321 1322 retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE); 1323 if (retval) 1324 return retval; 1325 1326 for (seg = 0; seg < nr_segs; seg++) { 1327 read_descriptor_t desc; 1328 1329 desc.written = 0; 1330 desc.arg.buf = iov[seg].iov_base; 1331 desc.count = iov[seg].iov_len; 1332 if (desc.count == 0) 1333 continue; 1334 desc.error = 0; 1335 do_shmem_file_read(filp, ppos, &desc, file_read_actor); 1336 retval += desc.written; 1337 if (desc.error) { 1338 retval = retval ?: desc.error; 1339 break; 1340 } 1341 if (desc.count > 0) 1342 break; 1343 } 1344 return retval; 1345} 1346 1347static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos, 1348 struct pipe_inode_info *pipe, size_t len, 1349 unsigned int flags) 1350{ 1351 struct address_space *mapping = in->f_mapping; 1352 struct inode *inode = mapping->host; 1353 unsigned int loff, nr_pages, req_pages; 1354 struct page *pages[PIPE_DEF_BUFFERS]; 1355 struct partial_page partial[PIPE_DEF_BUFFERS]; 1356 struct page *page; 1357 pgoff_t index, end_index; 1358 loff_t isize, left; 1359 int error, page_nr; 1360 struct splice_pipe_desc spd = { 1361 .pages = pages, 1362 .partial = partial, 1363 .flags = flags, 1364 .ops = &page_cache_pipe_buf_ops, 1365 .spd_release = spd_release_page, 1366 }; 1367 1368 isize = i_size_read(inode); 1369 if (unlikely(*ppos >= isize)) 1370 return 0; 1371 1372 left = isize - *ppos; 1373 if (unlikely(left < len)) 1374 len = left; 1375 1376 if (splice_grow_spd(pipe, &spd)) 1377 return -ENOMEM; 1378 1379 index = *ppos >> PAGE_CACHE_SHIFT; 1380 loff = *ppos & ~PAGE_CACHE_MASK; 1381 req_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 1382 nr_pages = min(req_pages, pipe->buffers); 1383 1384 spd.nr_pages = find_get_pages_contig(mapping, index, 1385 nr_pages, spd.pages); 1386 index += spd.nr_pages; 1387 error = 0; 1388 1389 while (spd.nr_pages < nr_pages) { 1390 error = shmem_getpage(inode, index, &page, SGP_CACHE, NULL); 1391 if (error) 1392 break; 1393 unlock_page(page); 1394 spd.pages[spd.nr_pages++] = page; 1395 index++; 1396 } 1397 1398 index = *ppos >> PAGE_CACHE_SHIFT; 1399 nr_pages = spd.nr_pages; 1400 spd.nr_pages = 0; 1401 1402 for (page_nr = 0; page_nr < nr_pages; page_nr++) { 1403 unsigned int this_len; 1404 1405 if (!len) 1406 break; 1407 1408 this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff); 1409 page = spd.pages[page_nr]; 1410 1411 if (!PageUptodate(page) || page->mapping != mapping) { 1412 error = shmem_getpage(inode, index, &page, 1413 SGP_CACHE, NULL); 1414 if (error) 1415 break; 1416 unlock_page(page); 1417 page_cache_release(spd.pages[page_nr]); 1418 spd.pages[page_nr] = page; 1419 } 1420 1421 isize = i_size_read(inode); 1422 end_index = (isize - 1) >> PAGE_CACHE_SHIFT; 1423 if (unlikely(!isize || index > end_index)) 1424 break; 1425 1426 if (end_index == index) { 1427 unsigned int plen; 1428 1429 plen = ((isize - 1) & ~PAGE_CACHE_MASK) + 1; 1430 if (plen <= loff) 1431 break; 1432 1433 this_len = min(this_len, plen - loff); 1434 len = this_len; 1435 } 1436 1437 spd.partial[page_nr].offset = loff; 1438 spd.partial[page_nr].len = this_len; 1439 len -= this_len; 1440 loff = 0; 1441 spd.nr_pages++; 1442 index++; 1443 } 1444 1445 while (page_nr < nr_pages) 1446 page_cache_release(spd.pages[page_nr++]); 1447 1448 if (spd.nr_pages) 1449 error = splice_to_pipe(pipe, &spd); 1450 1451 splice_shrink_spd(pipe, &spd); 1452 1453 if (error > 0) { 1454 *ppos += error; 1455 file_accessed(in); 1456 } 1457 return error; 1458} 1459 1460static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf) 1461{ 1462 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb); 1463 1464 buf->f_type = TMPFS_MAGIC; 1465 buf->f_bsize = PAGE_CACHE_SIZE; 1466 buf->f_namelen = NAME_MAX; 1467 if (sbinfo->max_blocks) { 1468 buf->f_blocks = sbinfo->max_blocks; 1469 buf->f_bavail = 1470 buf->f_bfree = sbinfo->max_blocks - 1471 percpu_counter_sum(&sbinfo->used_blocks); 1472 } 1473 if (sbinfo->max_inodes) { 1474 buf->f_files = sbinfo->max_inodes; 1475 buf->f_ffree = sbinfo->free_inodes; 1476 } 1477 /* else leave those fields 0 like simple_statfs */ 1478 return 0; 1479} 1480 1481/* 1482 * File creation. Allocate an inode, and we're done.. 1483 */ 1484static int 1485shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev) 1486{ 1487 struct inode *inode; 1488 int error = -ENOSPC; 1489 1490 inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE); 1491 if (inode) { 1492 error = security_inode_init_security(inode, dir, 1493 &dentry->d_name, NULL, 1494 NULL, NULL); 1495 if (error) { 1496 if (error != -EOPNOTSUPP) { 1497 iput(inode); 1498 return error; 1499 } 1500 } 1501#ifdef CONFIG_TMPFS_POSIX_ACL 1502 error = generic_acl_init(inode, dir); 1503 if (error) { 1504 iput(inode); 1505 return error; 1506 } 1507#else 1508 error = 0; 1509#endif 1510 dir->i_size += BOGO_DIRENT_SIZE; 1511 dir->i_ctime = dir->i_mtime = CURRENT_TIME; 1512 d_instantiate(dentry, inode); 1513 dget(dentry); /* Extra count - pin the dentry in core */ 1514 } 1515 return error; 1516} 1517 1518static int shmem_mkdir(struct inode *dir, struct dentry *dentry, int mode) 1519{ 1520 int error; 1521 1522 if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0))) 1523 return error; 1524 inc_nlink(dir); 1525 return 0; 1526} 1527 1528static int shmem_create(struct inode *dir, struct dentry *dentry, int mode, 1529 struct nameidata *nd) 1530{ 1531 return shmem_mknod(dir, dentry, mode | S_IFREG, 0); 1532} 1533 1534/* 1535 * Link a file.. 1536 */ 1537static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) 1538{ 1539 struct inode *inode = old_dentry->d_inode; 1540 int ret; 1541 1542 /* 1543 * No ordinary (disk based) filesystem counts links as inodes; 1544 * but each new link needs a new dentry, pinning lowmem, and 1545 * tmpfs dentries cannot be pruned until they are unlinked. 1546 */ 1547 ret = shmem_reserve_inode(inode->i_sb); 1548 if (ret) 1549 goto out; 1550 1551 dir->i_size += BOGO_DIRENT_SIZE; 1552 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME; 1553 inc_nlink(inode); 1554 ihold(inode); /* New dentry reference */ 1555 dget(dentry); /* Extra pinning count for the created dentry */ 1556 d_instantiate(dentry, inode); 1557out: 1558 return ret; 1559} 1560 1561static int shmem_unlink(struct inode *dir, struct dentry *dentry) 1562{ 1563 struct inode *inode = dentry->d_inode; 1564 1565 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode)) 1566 shmem_free_inode(inode->i_sb); 1567 1568 dir->i_size -= BOGO_DIRENT_SIZE; 1569 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME; 1570 drop_nlink(inode); 1571 dput(dentry); /* Undo the count from "create" - this does all the work */ 1572 return 0; 1573} 1574 1575static int shmem_rmdir(struct inode *dir, struct dentry *dentry) 1576{ 1577 if (!simple_empty(dentry)) 1578 return -ENOTEMPTY; 1579 1580 drop_nlink(dentry->d_inode); 1581 drop_nlink(dir); 1582 return shmem_unlink(dir, dentry); 1583} 1584 1585/* 1586 * The VFS layer already does all the dentry stuff for rename, 1587 * we just have to decrement the usage count for the target if 1588 * it exists so that the VFS layer correctly free's it when it 1589 * gets overwritten. 1590 */ 1591static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry) 1592{ 1593 struct inode *inode = old_dentry->d_inode; 1594 int they_are_dirs = S_ISDIR(inode->i_mode); 1595 1596 if (!simple_empty(new_dentry)) 1597 return -ENOTEMPTY; 1598 1599 if (new_dentry->d_inode) { 1600 (void) shmem_unlink(new_dir, new_dentry); 1601 if (they_are_dirs) 1602 drop_nlink(old_dir); 1603 } else if (they_are_dirs) { 1604 drop_nlink(old_dir); 1605 inc_nlink(new_dir); 1606 } 1607 1608 old_dir->i_size -= BOGO_DIRENT_SIZE; 1609 new_dir->i_size += BOGO_DIRENT_SIZE; 1610 old_dir->i_ctime = old_dir->i_mtime = 1611 new_dir->i_ctime = new_dir->i_mtime = 1612 inode->i_ctime = CURRENT_TIME; 1613 return 0; 1614} 1615 1616static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname) 1617{ 1618 int error; 1619 int len; 1620 struct inode *inode; 1621 struct page *page; 1622 char *kaddr; 1623 struct shmem_inode_info *info; 1624 1625 len = strlen(symname) + 1; 1626 if (len > PAGE_CACHE_SIZE) 1627 return -ENAMETOOLONG; 1628 1629 inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE); 1630 if (!inode) 1631 return -ENOSPC; 1632 1633 error = security_inode_init_security(inode, dir, &dentry->d_name, NULL, 1634 NULL, NULL); 1635 if (error) { 1636 if (error != -EOPNOTSUPP) { 1637 iput(inode); 1638 return error; 1639 } 1640 error = 0; 1641 } 1642 1643 info = SHMEM_I(inode); 1644 inode->i_size = len-1; 1645 if (len <= SHORT_SYMLINK_LEN) { 1646 info->symlink = kmemdup(symname, len, GFP_KERNEL); 1647 if (!info->symlink) { 1648 iput(inode); 1649 return -ENOMEM; 1650 } 1651 inode->i_op = &shmem_short_symlink_operations; 1652 } else { 1653 error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL); 1654 if (error) { 1655 iput(inode); 1656 return error; 1657 } 1658 inode->i_mapping->a_ops = &shmem_aops; 1659 inode->i_op = &shmem_symlink_inode_operations; 1660 kaddr = kmap_atomic(page, KM_USER0); 1661 memcpy(kaddr, symname, len); 1662 kunmap_atomic(kaddr, KM_USER0); 1663 set_page_dirty(page); 1664 unlock_page(page); 1665 page_cache_release(page); 1666 } 1667 dir->i_size += BOGO_DIRENT_SIZE; 1668 dir->i_ctime = dir->i_mtime = CURRENT_TIME; 1669 d_instantiate(dentry, inode); 1670 dget(dentry); 1671 return 0; 1672} 1673 1674static void *shmem_follow_short_symlink(struct dentry *dentry, struct nameidata *nd) 1675{ 1676 nd_set_link(nd, SHMEM_I(dentry->d_inode)->symlink); 1677 return NULL; 1678} 1679 1680static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd) 1681{ 1682 struct page *page = NULL; 1683 int error = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL); 1684 nd_set_link(nd, error ? ERR_PTR(error) : kmap(page)); 1685 if (page) 1686 unlock_page(page); 1687 return page; 1688} 1689 1690static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie) 1691{ 1692 if (!IS_ERR(nd_get_link(nd))) { 1693 struct page *page = cookie; 1694 kunmap(page); 1695 mark_page_accessed(page); 1696 page_cache_release(page); 1697 } 1698} 1699 1700#ifdef CONFIG_TMPFS_XATTR 1701/* 1702 * Superblocks without xattr inode operations may get some security.* xattr 1703 * support from the LSM "for free". As soon as we have any other xattrs 1704 * like ACLs, we also need to implement the security.* handlers at 1705 * filesystem level, though. 1706 */ 1707 1708static int shmem_xattr_get(struct dentry *dentry, const char *name, 1709 void *buffer, size_t size) 1710{ 1711 struct shmem_inode_info *info; 1712 struct shmem_xattr *xattr; 1713 int ret = -ENODATA; 1714 1715 info = SHMEM_I(dentry->d_inode); 1716 1717 spin_lock(&info->lock); 1718 list_for_each_entry(xattr, &info->xattr_list, list) { 1719 if (strcmp(name, xattr->name)) 1720 continue; 1721 1722 ret = xattr->size; 1723 if (buffer) { 1724 if (size < xattr->size) 1725 ret = -ERANGE; 1726 else 1727 memcpy(buffer, xattr->value, xattr->size); 1728 } 1729 break; 1730 } 1731 spin_unlock(&info->lock); 1732 return ret; 1733} 1734 1735static int shmem_xattr_set(struct dentry *dentry, const char *name, 1736 const void *value, size_t size, int flags) 1737{ 1738 struct inode *inode = dentry->d_inode; 1739 struct shmem_inode_info *info = SHMEM_I(inode); 1740 struct shmem_xattr *xattr; 1741 struct shmem_xattr *new_xattr = NULL; 1742 size_t len; 1743 int err = 0; 1744 1745 /* value == NULL means remove */ 1746 if (value) { 1747 /* wrap around? */ 1748 len = sizeof(*new_xattr) + size; 1749 if (len <= sizeof(*new_xattr)) 1750 return -ENOMEM; 1751 1752 new_xattr = kmalloc(len, GFP_KERNEL); 1753 if (!new_xattr) 1754 return -ENOMEM; 1755 1756 new_xattr->name = kstrdup(name, GFP_KERNEL); 1757 if (!new_xattr->name) { 1758 kfree(new_xattr); 1759 return -ENOMEM; 1760 } 1761 1762 new_xattr->size = size; 1763 memcpy(new_xattr->value, value, size); 1764 } 1765 1766 spin_lock(&info->lock); 1767 list_for_each_entry(xattr, &info->xattr_list, list) { 1768 if (!strcmp(name, xattr->name)) { 1769 if (flags & XATTR_CREATE) { 1770 xattr = new_xattr; 1771 err = -EEXIST; 1772 } else if (new_xattr) { 1773 list_replace(&xattr->list, &new_xattr->list); 1774 } else { 1775 list_del(&xattr->list); 1776 } 1777 goto out; 1778 } 1779 } 1780 if (flags & XATTR_REPLACE) { 1781 xattr = new_xattr; 1782 err = -ENODATA; 1783 } else { 1784 list_add(&new_xattr->list, &info->xattr_list); 1785 xattr = NULL; 1786 } 1787out: 1788 spin_unlock(&info->lock); 1789 if (xattr) 1790 kfree(xattr->name); 1791 kfree(xattr); 1792 return err; 1793} 1794 1795static const struct xattr_handler *shmem_xattr_handlers[] = { 1796#ifdef CONFIG_TMPFS_POSIX_ACL 1797 &generic_acl_access_handler, 1798 &generic_acl_default_handler, 1799#endif 1800 NULL 1801}; 1802 1803static int shmem_xattr_validate(const char *name) 1804{ 1805 struct { const char *prefix; size_t len; } arr[] = { 1806 { XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN }, 1807 { XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN } 1808 }; 1809 int i; 1810 1811 for (i = 0; i < ARRAY_SIZE(arr); i++) { 1812 size_t preflen = arr[i].len; 1813 if (strncmp(name, arr[i].prefix, preflen) == 0) { 1814 if (!name[preflen]) 1815 return -EINVAL; 1816 return 0; 1817 } 1818 } 1819 return -EOPNOTSUPP; 1820} 1821 1822static ssize_t shmem_getxattr(struct dentry *dentry, const char *name, 1823 void *buffer, size_t size) 1824{ 1825 int err; 1826 1827 /* 1828 * If this is a request for a synthetic attribute in the system.* 1829 * namespace use the generic infrastructure to resolve a handler 1830 * for it via sb->s_xattr. 1831 */ 1832 if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN)) 1833 return generic_getxattr(dentry, name, buffer, size); 1834 1835 err = shmem_xattr_validate(name); 1836 if (err) 1837 return err; 1838 1839 return shmem_xattr_get(dentry, name, buffer, size); 1840} 1841 1842static int shmem_setxattr(struct dentry *dentry, const char *name, 1843 const void *value, size_t size, int flags) 1844{ 1845 int err; 1846 1847 /* 1848 * If this is a request for a synthetic attribute in the system.* 1849 * namespace use the generic infrastructure to resolve a handler 1850 * for it via sb->s_xattr. 1851 */ 1852 if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN)) 1853 return generic_setxattr(dentry, name, value, size, flags); 1854 1855 err = shmem_xattr_validate(name); 1856 if (err) 1857 return err; 1858 1859 if (size == 0) 1860 value = ""; /* empty EA, do not remove */ 1861 1862 return shmem_xattr_set(dentry, name, value, size, flags); 1863 1864} 1865 1866static int shmem_removexattr(struct dentry *dentry, const char *name) 1867{ 1868 int err; 1869 1870 /* 1871 * If this is a request for a synthetic attribute in the system.* 1872 * namespace use the generic infrastructure to resolve a handler 1873 * for it via sb->s_xattr. 1874 */ 1875 if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN)) 1876 return generic_removexattr(dentry, name); 1877 1878 err = shmem_xattr_validate(name); 1879 if (err) 1880 return err; 1881 1882 return shmem_xattr_set(dentry, name, NULL, 0, XATTR_REPLACE); 1883} 1884 1885static bool xattr_is_trusted(const char *name) 1886{ 1887 return !strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN); 1888} 1889 1890static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size) 1891{ 1892 bool trusted = capable(CAP_SYS_ADMIN); 1893 struct shmem_xattr *xattr; 1894 struct shmem_inode_info *info; 1895 size_t used = 0; 1896 1897 info = SHMEM_I(dentry->d_inode); 1898 1899 spin_lock(&info->lock); 1900 list_for_each_entry(xattr, &info->xattr_list, list) { 1901 size_t len; 1902 1903 /* skip "trusted." attributes for unprivileged callers */ 1904 if (!trusted && xattr_is_trusted(xattr->name)) 1905 continue; 1906 1907 len = strlen(xattr->name) + 1; 1908 used += len; 1909 if (buffer) { 1910 if (size < used) { 1911 used = -ERANGE; 1912 break; 1913 } 1914 memcpy(buffer, xattr->name, len); 1915 buffer += len; 1916 } 1917 } 1918 spin_unlock(&info->lock); 1919 1920 return used; 1921} 1922#endif /* CONFIG_TMPFS_XATTR */ 1923 1924static const struct inode_operations shmem_short_symlink_operations = { 1925 .readlink = generic_readlink, 1926 .follow_link = shmem_follow_short_symlink, 1927#ifdef CONFIG_TMPFS_XATTR 1928 .setxattr = shmem_setxattr, 1929 .getxattr = shmem_getxattr, 1930 .listxattr = shmem_listxattr, 1931 .removexattr = shmem_removexattr, 1932#endif 1933}; 1934 1935static const struct inode_operations shmem_symlink_inode_operations = { 1936 .readlink = generic_readlink, 1937 .follow_link = shmem_follow_link, 1938 .put_link = shmem_put_link, 1939#ifdef CONFIG_TMPFS_XATTR 1940 .setxattr = shmem_setxattr, 1941 .getxattr = shmem_getxattr, 1942 .listxattr = shmem_listxattr, 1943 .removexattr = shmem_removexattr, 1944#endif 1945}; 1946 1947static struct dentry *shmem_get_parent(struct dentry *child) 1948{ 1949 return ERR_PTR(-ESTALE); 1950} 1951 1952static int shmem_match(struct inode *ino, void *vfh) 1953{ 1954 __u32 *fh = vfh; 1955 __u64 inum = fh[2]; 1956 inum = (inum << 32) | fh[1]; 1957 return ino->i_ino == inum && fh[0] == ino->i_generation; 1958} 1959 1960static struct dentry *shmem_fh_to_dentry(struct super_block *sb, 1961 struct fid *fid, int fh_len, int fh_type) 1962{ 1963 struct inode *inode; 1964 struct dentry *dentry = NULL; 1965 u64 inum = fid->raw[2]; 1966 inum = (inum << 32) | fid->raw[1]; 1967 1968 if (fh_len < 3) 1969 return NULL; 1970 1971 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]), 1972 shmem_match, fid->raw); 1973 if (inode) { 1974 dentry = d_find_alias(inode); 1975 iput(inode); 1976 } 1977 1978 return dentry; 1979} 1980 1981static int shmem_encode_fh(struct dentry *dentry, __u32 *fh, int *len, 1982 int connectable) 1983{ 1984 struct inode *inode = dentry->d_inode; 1985 1986 if (*len < 3) { 1987 *len = 3; 1988 return 255; 1989 } 1990 1991 if (inode_unhashed(inode)) { 1992 /* Unfortunately insert_inode_hash is not idempotent, 1993 * so as we hash inodes here rather than at creation 1994 * time, we need a lock to ensure we only try 1995 * to do it once 1996 */ 1997 static DEFINE_SPINLOCK(lock); 1998 spin_lock(&lock); 1999 if (inode_unhashed(inode)) 2000 __insert_inode_hash(inode, 2001 inode->i_ino + inode->i_generation); 2002 spin_unlock(&lock); 2003 } 2004 2005 fh[0] = inode->i_generation; 2006 fh[1] = inode->i_ino; 2007 fh[2] = ((__u64)inode->i_ino) >> 32; 2008 2009 *len = 3; 2010 return 1; 2011} 2012 2013static const struct export_operations shmem_export_ops = { 2014 .get_parent = shmem_get_parent, 2015 .encode_fh = shmem_encode_fh, 2016 .fh_to_dentry = shmem_fh_to_dentry, 2017}; 2018 2019static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo, 2020 bool remount) 2021{ 2022 char *this_char, *value, *rest; 2023 2024 while (options != NULL) { 2025 this_char = options; 2026 for (;;) { 2027 /* 2028 * NUL-terminate this option: unfortunately, 2029 * mount options form a comma-separated list, 2030 * but mpol's nodelist may also contain commas. 2031 */ 2032 options = strchr(options, ','); 2033 if (options == NULL) 2034 break; 2035 options++; 2036 if (!isdigit(*options)) { 2037 options[-1] = '\0'; 2038 break; 2039 } 2040 } 2041 if (!*this_char) 2042 continue; 2043 if ((value = strchr(this_char,'=')) != NULL) { 2044 *value++ = 0; 2045 } else { 2046 printk(KERN_ERR 2047 "tmpfs: No value for mount option '%s'\n", 2048 this_char); 2049 return 1; 2050 } 2051 2052 if (!strcmp(this_char,"size")) { 2053 unsigned long long size; 2054 size = memparse(value,&rest); 2055 if (*rest == '%') { 2056 size <<= PAGE_SHIFT; 2057 size *= totalram_pages; 2058 do_div(size, 100); 2059 rest++; 2060 } 2061 if (*rest) 2062 goto bad_val; 2063 sbinfo->max_blocks = 2064 DIV_ROUND_UP(size, PAGE_CACHE_SIZE); 2065 } else if (!strcmp(this_char,"nr_blocks")) { 2066 sbinfo->max_blocks = memparse(value, &rest); 2067 if (*rest) 2068 goto bad_val; 2069 } else if (!strcmp(this_char,"nr_inodes")) { 2070 sbinfo->max_inodes = memparse(value, &rest); 2071 if (*rest) 2072 goto bad_val; 2073 } else if (!strcmp(this_char,"mode")) { 2074 if (remount) 2075 continue; 2076 sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777; 2077 if (*rest) 2078 goto bad_val; 2079 } else if (!strcmp(this_char,"uid")) { 2080 if (remount) 2081 continue; 2082 sbinfo->uid = simple_strtoul(value, &rest, 0); 2083 if (*rest) 2084 goto bad_val; 2085 } else if (!strcmp(this_char,"gid")) { 2086 if (remount) 2087 continue; 2088 sbinfo->gid = simple_strtoul(value, &rest, 0); 2089 if (*rest) 2090 goto bad_val; 2091 } else if (!strcmp(this_char,"mpol")) { 2092 if (mpol_parse_str(value, &sbinfo->mpol, 1)) 2093 goto bad_val; 2094 } else { 2095 printk(KERN_ERR "tmpfs: Bad mount option %s\n", 2096 this_char); 2097 return 1; 2098 } 2099 } 2100 return 0; 2101 2102bad_val: 2103 printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n", 2104 value, this_char); 2105 return 1; 2106 2107} 2108 2109static int shmem_remount_fs(struct super_block *sb, int *flags, char *data) 2110{ 2111 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 2112 struct shmem_sb_info config = *sbinfo; 2113 unsigned long inodes; 2114 int error = -EINVAL; 2115 2116 if (shmem_parse_options(data, &config, true)) 2117 return error; 2118 2119 spin_lock(&sbinfo->stat_lock); 2120 inodes = sbinfo->max_inodes - sbinfo->free_inodes; 2121 if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0) 2122 goto out; 2123 if (config.max_inodes < inodes) 2124 goto out; 2125 /* 2126 * Those tests disallow limited->unlimited while any are in use; 2127 * but we must separately disallow unlimited->limited, because 2128 * in that case we have no record of how much is already in use. 2129 */ 2130 if (config.max_blocks && !sbinfo->max_blocks) 2131 goto out; 2132 if (config.max_inodes && !sbinfo->max_inodes) 2133 goto out; 2134 2135 error = 0; 2136 sbinfo->max_blocks = config.max_blocks; 2137 sbinfo->max_inodes = config.max_inodes; 2138 sbinfo->free_inodes = config.max_inodes - inodes; 2139 2140 mpol_put(sbinfo->mpol); 2141 sbinfo->mpol = config.mpol; /* transfers initial ref */ 2142out: 2143 spin_unlock(&sbinfo->stat_lock); 2144 return error; 2145} 2146 2147static int shmem_show_options(struct seq_file *seq, struct vfsmount *vfs) 2148{ 2149 struct shmem_sb_info *sbinfo = SHMEM_SB(vfs->mnt_sb); 2150 2151 if (sbinfo->max_blocks != shmem_default_max_blocks()) 2152 seq_printf(seq, ",size=%luk", 2153 sbinfo->max_blocks << (PAGE_CACHE_SHIFT - 10)); 2154 if (sbinfo->max_inodes != shmem_default_max_inodes()) 2155 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes); 2156 if (sbinfo->mode != (S_IRWXUGO | S_ISVTX)) 2157 seq_printf(seq, ",mode=%03o", sbinfo->mode); 2158 if (sbinfo->uid != 0) 2159 seq_printf(seq, ",uid=%u", sbinfo->uid); 2160 if (sbinfo->gid != 0) 2161 seq_printf(seq, ",gid=%u", sbinfo->gid); 2162 shmem_show_mpol(seq, sbinfo->mpol); 2163 return 0; 2164} 2165#endif /* CONFIG_TMPFS */ 2166 2167static void shmem_put_super(struct super_block *sb) 2168{ 2169 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 2170 2171 percpu_counter_destroy(&sbinfo->used_blocks); 2172 kfree(sbinfo); 2173 sb->s_fs_info = NULL; 2174} 2175 2176int shmem_fill_super(struct super_block *sb, void *data, int silent) 2177{ 2178 struct inode *inode; 2179 struct dentry *root; 2180 struct shmem_sb_info *sbinfo; 2181 int err = -ENOMEM; 2182 2183 /* Round up to L1_CACHE_BYTES to resist false sharing */ 2184 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info), 2185 L1_CACHE_BYTES), GFP_KERNEL); 2186 if (!sbinfo) 2187 return -ENOMEM; 2188 2189 sbinfo->mode = S_IRWXUGO | S_ISVTX; 2190 sbinfo->uid = current_fsuid(); 2191 sbinfo->gid = current_fsgid(); 2192 sb->s_fs_info = sbinfo; 2193 2194#ifdef CONFIG_TMPFS 2195 /* 2196 * Per default we only allow half of the physical ram per 2197 * tmpfs instance, limiting inodes to one per page of lowmem; 2198 * but the internal instance is left unlimited. 2199 */ 2200 if (!(sb->s_flags & MS_NOUSER)) { 2201 sbinfo->max_blocks = shmem_default_max_blocks(); 2202 sbinfo->max_inodes = shmem_default_max_inodes(); 2203 if (shmem_parse_options(data, sbinfo, false)) { 2204 err = -EINVAL; 2205 goto failed; 2206 } 2207 } 2208 sb->s_export_op = &shmem_export_ops; 2209#else 2210 sb->s_flags |= MS_NOUSER; 2211#endif 2212 2213 spin_lock_init(&sbinfo->stat_lock); 2214 if (percpu_counter_init(&sbinfo->used_blocks, 0)) 2215 goto failed; 2216 sbinfo->free_inodes = sbinfo->max_inodes; 2217 2218 sb->s_maxbytes = MAX_LFS_FILESIZE; 2219 sb->s_blocksize = PAGE_CACHE_SIZE; 2220 sb->s_blocksize_bits = PAGE_CACHE_SHIFT; 2221 sb->s_magic = TMPFS_MAGIC; 2222 sb->s_op = &shmem_ops; 2223 sb->s_time_gran = 1; 2224#ifdef CONFIG_TMPFS_XATTR 2225 sb->s_xattr = shmem_xattr_handlers; 2226#endif 2227#ifdef CONFIG_TMPFS_POSIX_ACL 2228 sb->s_flags |= MS_POSIXACL; 2229#endif 2230 2231 inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE); 2232 if (!inode) 2233 goto failed; 2234 inode->i_uid = sbinfo->uid; 2235 inode->i_gid = sbinfo->gid; 2236 root = d_alloc_root(inode); 2237 if (!root) 2238 goto failed_iput; 2239 sb->s_root = root; 2240 return 0; 2241 2242failed_iput: 2243 iput(inode); 2244failed: 2245 shmem_put_super(sb); 2246 return err; 2247} 2248 2249static struct kmem_cache *shmem_inode_cachep; 2250 2251static struct inode *shmem_alloc_inode(struct super_block *sb) 2252{ 2253 struct shmem_inode_info *info; 2254 info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL); 2255 if (!info) 2256 return NULL; 2257 return &info->vfs_inode; 2258} 2259 2260static void shmem_destroy_callback(struct rcu_head *head) 2261{ 2262 struct inode *inode = container_of(head, struct inode, i_rcu); 2263 INIT_LIST_HEAD(&inode->i_dentry); 2264 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode)); 2265} 2266 2267static void shmem_destroy_inode(struct inode *inode) 2268{ 2269 if ((inode->i_mode & S_IFMT) == S_IFREG) 2270 mpol_free_shared_policy(&SHMEM_I(inode)->policy); 2271 call_rcu(&inode->i_rcu, shmem_destroy_callback); 2272} 2273 2274static void shmem_init_inode(void *foo) 2275{ 2276 struct shmem_inode_info *info = foo; 2277 inode_init_once(&info->vfs_inode); 2278} 2279 2280static int shmem_init_inodecache(void) 2281{ 2282 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache", 2283 sizeof(struct shmem_inode_info), 2284 0, SLAB_PANIC, shmem_init_inode); 2285 return 0; 2286} 2287 2288static void shmem_destroy_inodecache(void) 2289{ 2290 kmem_cache_destroy(shmem_inode_cachep); 2291} 2292 2293static const struct address_space_operations shmem_aops = { 2294 .writepage = shmem_writepage, 2295 .set_page_dirty = __set_page_dirty_no_writeback, 2296#ifdef CONFIG_TMPFS 2297 .write_begin = shmem_write_begin, 2298 .write_end = shmem_write_end, 2299#endif 2300 .migratepage = migrate_page, 2301 .error_remove_page = generic_error_remove_page, 2302}; 2303 2304static const struct file_operations shmem_file_operations = { 2305 .mmap = shmem_mmap, 2306#ifdef CONFIG_TMPFS 2307 .llseek = generic_file_llseek, 2308 .read = do_sync_read, 2309 .write = do_sync_write, 2310 .aio_read = shmem_file_aio_read, 2311 .aio_write = generic_file_aio_write, 2312 .fsync = noop_fsync, 2313 .splice_read = shmem_file_splice_read, 2314 .splice_write = generic_file_splice_write, 2315#endif 2316}; 2317 2318static const struct inode_operations shmem_inode_operations = { 2319 .setattr = shmem_setattr, 2320 .truncate_range = shmem_truncate_range, 2321#ifdef CONFIG_TMPFS_XATTR 2322 .setxattr = shmem_setxattr, 2323 .getxattr = shmem_getxattr, 2324 .listxattr = shmem_listxattr, 2325 .removexattr = shmem_removexattr, 2326#endif 2327}; 2328 2329static const struct inode_operations shmem_dir_inode_operations = { 2330#ifdef CONFIG_TMPFS 2331 .create = shmem_create, 2332 .lookup = simple_lookup, 2333 .link = shmem_link, 2334 .unlink = shmem_unlink, 2335 .symlink = shmem_symlink, 2336 .mkdir = shmem_mkdir, 2337 .rmdir = shmem_rmdir, 2338 .mknod = shmem_mknod, 2339 .rename = shmem_rename, 2340#endif 2341#ifdef CONFIG_TMPFS_XATTR 2342 .setxattr = shmem_setxattr, 2343 .getxattr = shmem_getxattr, 2344 .listxattr = shmem_listxattr, 2345 .removexattr = shmem_removexattr, 2346#endif 2347#ifdef CONFIG_TMPFS_POSIX_ACL 2348 .setattr = shmem_setattr, 2349#endif 2350}; 2351 2352static const struct inode_operations shmem_special_inode_operations = { 2353#ifdef CONFIG_TMPFS_XATTR 2354 .setxattr = shmem_setxattr, 2355 .getxattr = shmem_getxattr, 2356 .listxattr = shmem_listxattr, 2357 .removexattr = shmem_removexattr, 2358#endif 2359#ifdef CONFIG_TMPFS_POSIX_ACL 2360 .setattr = shmem_setattr, 2361#endif 2362}; 2363 2364static const struct super_operations shmem_ops = { 2365 .alloc_inode = shmem_alloc_inode, 2366 .destroy_inode = shmem_destroy_inode, 2367#ifdef CONFIG_TMPFS 2368 .statfs = shmem_statfs, 2369 .remount_fs = shmem_remount_fs, 2370 .show_options = shmem_show_options, 2371#endif 2372 .evict_inode = shmem_evict_inode, 2373 .drop_inode = generic_delete_inode, 2374 .put_super = shmem_put_super, 2375}; 2376 2377static const struct vm_operations_struct shmem_vm_ops = { 2378 .fault = shmem_fault, 2379#ifdef CONFIG_NUMA 2380 .set_policy = shmem_set_policy, 2381 .get_policy = shmem_get_policy, 2382#endif 2383}; 2384 2385static struct dentry *shmem_mount(struct file_system_type *fs_type, 2386 int flags, const char *dev_name, void *data) 2387{ 2388 return mount_nodev(fs_type, flags, data, shmem_fill_super); 2389} 2390 2391static struct file_system_type shmem_fs_type = { 2392 .owner = THIS_MODULE, 2393 .name = "tmpfs", 2394 .mount = shmem_mount, 2395 .kill_sb = kill_litter_super, 2396}; 2397 2398int __init shmem_init(void) 2399{ 2400 int error; 2401 2402 error = bdi_init(&shmem_backing_dev_info); 2403 if (error) 2404 goto out4; 2405 2406 error = shmem_init_inodecache(); 2407 if (error) 2408 goto out3; 2409 2410 error = register_filesystem(&shmem_fs_type); 2411 if (error) { 2412 printk(KERN_ERR "Could not register tmpfs\n"); 2413 goto out2; 2414 } 2415 2416 shm_mnt = vfs_kern_mount(&shmem_fs_type, MS_NOUSER, 2417 shmem_fs_type.name, NULL); 2418 if (IS_ERR(shm_mnt)) { 2419 error = PTR_ERR(shm_mnt); 2420 printk(KERN_ERR "Could not kern_mount tmpfs\n"); 2421 goto out1; 2422 } 2423 return 0; 2424 2425out1: 2426 unregister_filesystem(&shmem_fs_type); 2427out2: 2428 shmem_destroy_inodecache(); 2429out3: 2430 bdi_destroy(&shmem_backing_dev_info); 2431out4: 2432 shm_mnt = ERR_PTR(error); 2433 return error; 2434} 2435 2436#else /* !CONFIG_SHMEM */ 2437 2438/* 2439 * tiny-shmem: simple shmemfs and tmpfs using ramfs code 2440 * 2441 * This is intended for small system where the benefits of the full 2442 * shmem code (swap-backed and resource-limited) are outweighed by 2443 * their complexity. On systems without swap this code should be 2444 * effectively equivalent, but much lighter weight. 2445 */ 2446 2447#include <linux/ramfs.h> 2448 2449static struct file_system_type shmem_fs_type = { 2450 .name = "tmpfs", 2451 .mount = ramfs_mount, 2452 .kill_sb = kill_litter_super, 2453}; 2454 2455int __init shmem_init(void) 2456{ 2457 BUG_ON(register_filesystem(&shmem_fs_type) != 0); 2458 2459 shm_mnt = kern_mount(&shmem_fs_type); 2460 BUG_ON(IS_ERR(shm_mnt)); 2461 2462 return 0; 2463} 2464 2465int shmem_unuse(swp_entry_t swap, struct page *page) 2466{ 2467 return 0; 2468} 2469 2470int shmem_lock(struct file *file, int lock, struct user_struct *user) 2471{ 2472 return 0; 2473} 2474 2475void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 2476{ 2477 truncate_inode_pages_range(inode->i_mapping, lstart, lend); 2478} 2479EXPORT_SYMBOL_GPL(shmem_truncate_range); 2480 2481#define shmem_vm_ops generic_file_vm_ops 2482#define shmem_file_operations ramfs_file_operations 2483#define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev) 2484#define shmem_acct_size(flags, size) 0 2485#define shmem_unacct_size(flags, size) do {} while (0) 2486 2487#endif /* CONFIG_SHMEM */ 2488 2489/* common code */ 2490 2491/** 2492 * shmem_file_setup - get an unlinked file living in tmpfs 2493 * @name: name for dentry (to be seen in /proc/<pid>/maps 2494 * @size: size to be set for the file 2495 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 2496 */ 2497struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags) 2498{ 2499 int error; 2500 struct file *file; 2501 struct inode *inode; 2502 struct path path; 2503 struct dentry *root; 2504 struct qstr this; 2505 2506 if (IS_ERR(shm_mnt)) 2507 return (void *)shm_mnt; 2508 2509 if (size < 0 || size > MAX_LFS_FILESIZE) 2510 return ERR_PTR(-EINVAL); 2511 2512 if (shmem_acct_size(flags, size)) 2513 return ERR_PTR(-ENOMEM); 2514 2515 error = -ENOMEM; 2516 this.name = name; 2517 this.len = strlen(name); 2518 this.hash = 0; /* will go */ 2519 root = shm_mnt->mnt_root; 2520 path.dentry = d_alloc(root, &this); 2521 if (!path.dentry) 2522 goto put_memory; 2523 path.mnt = mntget(shm_mnt); 2524 2525 error = -ENOSPC; 2526 inode = shmem_get_inode(root->d_sb, NULL, S_IFREG | S_IRWXUGO, 0, flags); 2527 if (!inode) 2528 goto put_dentry; 2529 2530 d_instantiate(path.dentry, inode); 2531 inode->i_size = size; 2532 inode->i_nlink = 0; /* It is unlinked */ 2533#ifndef CONFIG_MMU 2534 error = ramfs_nommu_expand_for_mapping(inode, size); 2535 if (error) 2536 goto put_dentry; 2537#endif 2538 2539 error = -ENFILE; 2540 file = alloc_file(&path, FMODE_WRITE | FMODE_READ, 2541 &shmem_file_operations); 2542 if (!file) 2543 goto put_dentry; 2544 2545 return file; 2546 2547put_dentry: 2548 path_put(&path); 2549put_memory: 2550 shmem_unacct_size(flags, size); 2551 return ERR_PTR(error); 2552} 2553EXPORT_SYMBOL_GPL(shmem_file_setup); 2554 2555/** 2556 * shmem_zero_setup - setup a shared anonymous mapping 2557 * @vma: the vma to be mmapped is prepared by do_mmap_pgoff 2558 */ 2559int shmem_zero_setup(struct vm_area_struct *vma) 2560{ 2561 struct file *file; 2562 loff_t size = vma->vm_end - vma->vm_start; 2563 2564 file = shmem_file_setup("dev/zero", size, vma->vm_flags); 2565 if (IS_ERR(file)) 2566 return PTR_ERR(file); 2567 2568 if (vma->vm_file) 2569 fput(vma->vm_file); 2570 vma->vm_file = file; 2571 vma->vm_ops = &shmem_vm_ops; 2572 vma->vm_flags |= VM_CAN_NONLINEAR; 2573 return 0; 2574} 2575 2576/** 2577 * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags. 2578 * @mapping: the page's address_space 2579 * @index: the page index 2580 * @gfp: the page allocator flags to use if allocating 2581 * 2582 * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)", 2583 * with any new page allocations done using the specified allocation flags. 2584 * But read_cache_page_gfp() uses the ->readpage() method: which does not 2585 * suit tmpfs, since it may have pages in swapcache, and needs to find those 2586 * for itself; although drivers/gpu/drm i915 and ttm rely upon this support. 2587 * 2588 * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in 2589 * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily. 2590 */ 2591struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, 2592 pgoff_t index, gfp_t gfp) 2593{ 2594#ifdef CONFIG_SHMEM 2595 struct inode *inode = mapping->host; 2596 struct page *page; 2597 int error; 2598 2599 BUG_ON(mapping->a_ops != &shmem_aops); 2600 error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, gfp, NULL); 2601 if (error) 2602 page = ERR_PTR(error); 2603 else 2604 unlock_page(page); 2605 return page; 2606#else 2607 /* 2608 * The tiny !SHMEM case uses ramfs without swap 2609 */ 2610 return read_cache_page_gfp(mapping, index, gfp); 2611#endif 2612} 2613EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp); 2614