shmem.c revision b09e0fa4b4ea66266058eead43350bd7d55fec67
1/* 2 * Resizable virtual memory filesystem for Linux. 3 * 4 * Copyright (C) 2000 Linus Torvalds. 5 * 2000 Transmeta Corp. 6 * 2000-2001 Christoph Rohland 7 * 2000-2001 SAP AG 8 * 2002 Red Hat Inc. 9 * Copyright (C) 2002-2005 Hugh Dickins. 10 * Copyright (C) 2002-2005 VERITAS Software Corporation. 11 * Copyright (C) 2004 Andi Kleen, SuSE Labs 12 * 13 * Extended attribute support for tmpfs: 14 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net> 15 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com> 16 * 17 * tiny-shmem: 18 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com> 19 * 20 * This file is released under the GPL. 21 */ 22 23#include <linux/fs.h> 24#include <linux/init.h> 25#include <linux/vfs.h> 26#include <linux/mount.h> 27#include <linux/pagemap.h> 28#include <linux/file.h> 29#include <linux/mm.h> 30#include <linux/module.h> 31#include <linux/percpu_counter.h> 32#include <linux/swap.h> 33 34static struct vfsmount *shm_mnt; 35 36#ifdef CONFIG_SHMEM 37/* 38 * This virtual memory filesystem is heavily based on the ramfs. It 39 * extends ramfs by the ability to use swap and honor resource limits 40 * which makes it a completely usable filesystem. 41 */ 42 43#include <linux/xattr.h> 44#include <linux/exportfs.h> 45#include <linux/posix_acl.h> 46#include <linux/generic_acl.h> 47#include <linux/mman.h> 48#include <linux/string.h> 49#include <linux/slab.h> 50#include <linux/backing-dev.h> 51#include <linux/shmem_fs.h> 52#include <linux/writeback.h> 53#include <linux/blkdev.h> 54#include <linux/security.h> 55#include <linux/swapops.h> 56#include <linux/mempolicy.h> 57#include <linux/namei.h> 58#include <linux/ctype.h> 59#include <linux/migrate.h> 60#include <linux/highmem.h> 61#include <linux/seq_file.h> 62#include <linux/magic.h> 63 64#include <asm/uaccess.h> 65#include <asm/div64.h> 66#include <asm/pgtable.h> 67 68/* 69 * The maximum size of a shmem/tmpfs file is limited by the maximum size of 70 * its triple-indirect swap vector - see illustration at shmem_swp_entry(). 71 * 72 * With 4kB page size, maximum file size is just over 2TB on a 32-bit kernel, 73 * but one eighth of that on a 64-bit kernel. With 8kB page size, maximum 74 * file size is just over 4TB on a 64-bit kernel, but 16TB on a 32-bit kernel, 75 * MAX_LFS_FILESIZE being then more restrictive than swap vector layout. 76 * 77 * We use / and * instead of shifts in the definitions below, so that the swap 78 * vector can be tested with small even values (e.g. 20) for ENTRIES_PER_PAGE. 79 */ 80#define ENTRIES_PER_PAGE (PAGE_CACHE_SIZE/sizeof(unsigned long)) 81#define ENTRIES_PER_PAGEPAGE ((unsigned long long)ENTRIES_PER_PAGE*ENTRIES_PER_PAGE) 82 83#define SHMSWP_MAX_INDEX (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1)) 84#define SHMSWP_MAX_BYTES (SHMSWP_MAX_INDEX << PAGE_CACHE_SHIFT) 85 86#define SHMEM_MAX_BYTES min_t(unsigned long long, SHMSWP_MAX_BYTES, MAX_LFS_FILESIZE) 87#define SHMEM_MAX_INDEX ((unsigned long)((SHMEM_MAX_BYTES+1) >> PAGE_CACHE_SHIFT)) 88 89#define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512) 90#define VM_ACCT(size) (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT) 91 92/* info->flags needs VM_flags to handle pagein/truncate races efficiently */ 93#define SHMEM_PAGEIN VM_READ 94#define SHMEM_TRUNCATE VM_WRITE 95 96/* Definition to limit shmem_truncate's steps between cond_rescheds */ 97#define LATENCY_LIMIT 64 98 99/* Pretend that each entry is of this size in directory's i_size */ 100#define BOGO_DIRENT_SIZE 20 101 102struct shmem_xattr { 103 struct list_head list; /* anchored by shmem_inode_info->xattr_list */ 104 char *name; /* xattr name */ 105 size_t size; 106 char value[0]; 107}; 108 109/* Flag allocation requirements to shmem_getpage and shmem_swp_alloc */ 110enum sgp_type { 111 SGP_READ, /* don't exceed i_size, don't allocate page */ 112 SGP_CACHE, /* don't exceed i_size, may allocate page */ 113 SGP_DIRTY, /* like SGP_CACHE, but set new page dirty */ 114 SGP_WRITE, /* may exceed i_size, may allocate page */ 115}; 116 117#ifdef CONFIG_TMPFS 118static unsigned long shmem_default_max_blocks(void) 119{ 120 return totalram_pages / 2; 121} 122 123static unsigned long shmem_default_max_inodes(void) 124{ 125 return min(totalram_pages - totalhigh_pages, totalram_pages / 2); 126} 127#endif 128 129static int shmem_getpage(struct inode *inode, unsigned long idx, 130 struct page **pagep, enum sgp_type sgp, int *type); 131 132static inline struct page *shmem_dir_alloc(gfp_t gfp_mask) 133{ 134 /* 135 * The above definition of ENTRIES_PER_PAGE, and the use of 136 * BLOCKS_PER_PAGE on indirect pages, assume PAGE_CACHE_SIZE: 137 * might be reconsidered if it ever diverges from PAGE_SIZE. 138 * 139 * Mobility flags are masked out as swap vectors cannot move 140 */ 141 return alloc_pages((gfp_mask & ~GFP_MOVABLE_MASK) | __GFP_ZERO, 142 PAGE_CACHE_SHIFT-PAGE_SHIFT); 143} 144 145static inline void shmem_dir_free(struct page *page) 146{ 147 __free_pages(page, PAGE_CACHE_SHIFT-PAGE_SHIFT); 148} 149 150static struct page **shmem_dir_map(struct page *page) 151{ 152 return (struct page **)kmap_atomic(page, KM_USER0); 153} 154 155static inline void shmem_dir_unmap(struct page **dir) 156{ 157 kunmap_atomic(dir, KM_USER0); 158} 159 160static swp_entry_t *shmem_swp_map(struct page *page) 161{ 162 return (swp_entry_t *)kmap_atomic(page, KM_USER1); 163} 164 165static inline void shmem_swp_balance_unmap(void) 166{ 167 /* 168 * When passing a pointer to an i_direct entry, to code which 169 * also handles indirect entries and so will shmem_swp_unmap, 170 * we must arrange for the preempt count to remain in balance. 171 * What kmap_atomic of a lowmem page does depends on config 172 * and architecture, so pretend to kmap_atomic some lowmem page. 173 */ 174 (void) kmap_atomic(ZERO_PAGE(0), KM_USER1); 175} 176 177static inline void shmem_swp_unmap(swp_entry_t *entry) 178{ 179 kunmap_atomic(entry, KM_USER1); 180} 181 182static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb) 183{ 184 return sb->s_fs_info; 185} 186 187/* 188 * shmem_file_setup pre-accounts the whole fixed size of a VM object, 189 * for shared memory and for shared anonymous (/dev/zero) mappings 190 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1), 191 * consistent with the pre-accounting of private mappings ... 192 */ 193static inline int shmem_acct_size(unsigned long flags, loff_t size) 194{ 195 return (flags & VM_NORESERVE) ? 196 0 : security_vm_enough_memory_kern(VM_ACCT(size)); 197} 198 199static inline void shmem_unacct_size(unsigned long flags, loff_t size) 200{ 201 if (!(flags & VM_NORESERVE)) 202 vm_unacct_memory(VM_ACCT(size)); 203} 204 205/* 206 * ... whereas tmpfs objects are accounted incrementally as 207 * pages are allocated, in order to allow huge sparse files. 208 * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM, 209 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM. 210 */ 211static inline int shmem_acct_block(unsigned long flags) 212{ 213 return (flags & VM_NORESERVE) ? 214 security_vm_enough_memory_kern(VM_ACCT(PAGE_CACHE_SIZE)) : 0; 215} 216 217static inline void shmem_unacct_blocks(unsigned long flags, long pages) 218{ 219 if (flags & VM_NORESERVE) 220 vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE)); 221} 222 223static const struct super_operations shmem_ops; 224static const struct address_space_operations shmem_aops; 225static const struct file_operations shmem_file_operations; 226static const struct inode_operations shmem_inode_operations; 227static const struct inode_operations shmem_dir_inode_operations; 228static const struct inode_operations shmem_special_inode_operations; 229static const struct vm_operations_struct shmem_vm_ops; 230 231static struct backing_dev_info shmem_backing_dev_info __read_mostly = { 232 .ra_pages = 0, /* No readahead */ 233 .capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED, 234}; 235 236static LIST_HEAD(shmem_swaplist); 237static DEFINE_MUTEX(shmem_swaplist_mutex); 238 239static void shmem_free_blocks(struct inode *inode, long pages) 240{ 241 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 242 if (sbinfo->max_blocks) { 243 percpu_counter_add(&sbinfo->used_blocks, -pages); 244 spin_lock(&inode->i_lock); 245 inode->i_blocks -= pages*BLOCKS_PER_PAGE; 246 spin_unlock(&inode->i_lock); 247 } 248} 249 250static int shmem_reserve_inode(struct super_block *sb) 251{ 252 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 253 if (sbinfo->max_inodes) { 254 spin_lock(&sbinfo->stat_lock); 255 if (!sbinfo->free_inodes) { 256 spin_unlock(&sbinfo->stat_lock); 257 return -ENOSPC; 258 } 259 sbinfo->free_inodes--; 260 spin_unlock(&sbinfo->stat_lock); 261 } 262 return 0; 263} 264 265static void shmem_free_inode(struct super_block *sb) 266{ 267 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 268 if (sbinfo->max_inodes) { 269 spin_lock(&sbinfo->stat_lock); 270 sbinfo->free_inodes++; 271 spin_unlock(&sbinfo->stat_lock); 272 } 273} 274 275/** 276 * shmem_recalc_inode - recalculate the size of an inode 277 * @inode: inode to recalc 278 * 279 * We have to calculate the free blocks since the mm can drop 280 * undirtied hole pages behind our back. 281 * 282 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped 283 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped) 284 * 285 * It has to be called with the spinlock held. 286 */ 287static void shmem_recalc_inode(struct inode *inode) 288{ 289 struct shmem_inode_info *info = SHMEM_I(inode); 290 long freed; 291 292 freed = info->alloced - info->swapped - inode->i_mapping->nrpages; 293 if (freed > 0) { 294 info->alloced -= freed; 295 shmem_unacct_blocks(info->flags, freed); 296 shmem_free_blocks(inode, freed); 297 } 298} 299 300/** 301 * shmem_swp_entry - find the swap vector position in the info structure 302 * @info: info structure for the inode 303 * @index: index of the page to find 304 * @page: optional page to add to the structure. Has to be preset to 305 * all zeros 306 * 307 * If there is no space allocated yet it will return NULL when 308 * page is NULL, else it will use the page for the needed block, 309 * setting it to NULL on return to indicate that it has been used. 310 * 311 * The swap vector is organized the following way: 312 * 313 * There are SHMEM_NR_DIRECT entries directly stored in the 314 * shmem_inode_info structure. So small files do not need an addional 315 * allocation. 316 * 317 * For pages with index > SHMEM_NR_DIRECT there is the pointer 318 * i_indirect which points to a page which holds in the first half 319 * doubly indirect blocks, in the second half triple indirect blocks: 320 * 321 * For an artificial ENTRIES_PER_PAGE = 4 this would lead to the 322 * following layout (for SHMEM_NR_DIRECT == 16): 323 * 324 * i_indirect -> dir --> 16-19 325 * | +-> 20-23 326 * | 327 * +-->dir2 --> 24-27 328 * | +-> 28-31 329 * | +-> 32-35 330 * | +-> 36-39 331 * | 332 * +-->dir3 --> 40-43 333 * +-> 44-47 334 * +-> 48-51 335 * +-> 52-55 336 */ 337static swp_entry_t *shmem_swp_entry(struct shmem_inode_info *info, unsigned long index, struct page **page) 338{ 339 unsigned long offset; 340 struct page **dir; 341 struct page *subdir; 342 343 if (index < SHMEM_NR_DIRECT) { 344 shmem_swp_balance_unmap(); 345 return info->i_direct+index; 346 } 347 if (!info->i_indirect) { 348 if (page) { 349 info->i_indirect = *page; 350 *page = NULL; 351 } 352 return NULL; /* need another page */ 353 } 354 355 index -= SHMEM_NR_DIRECT; 356 offset = index % ENTRIES_PER_PAGE; 357 index /= ENTRIES_PER_PAGE; 358 dir = shmem_dir_map(info->i_indirect); 359 360 if (index >= ENTRIES_PER_PAGE/2) { 361 index -= ENTRIES_PER_PAGE/2; 362 dir += ENTRIES_PER_PAGE/2 + index/ENTRIES_PER_PAGE; 363 index %= ENTRIES_PER_PAGE; 364 subdir = *dir; 365 if (!subdir) { 366 if (page) { 367 *dir = *page; 368 *page = NULL; 369 } 370 shmem_dir_unmap(dir); 371 return NULL; /* need another page */ 372 } 373 shmem_dir_unmap(dir); 374 dir = shmem_dir_map(subdir); 375 } 376 377 dir += index; 378 subdir = *dir; 379 if (!subdir) { 380 if (!page || !(subdir = *page)) { 381 shmem_dir_unmap(dir); 382 return NULL; /* need a page */ 383 } 384 *dir = subdir; 385 *page = NULL; 386 } 387 shmem_dir_unmap(dir); 388 return shmem_swp_map(subdir) + offset; 389} 390 391static void shmem_swp_set(struct shmem_inode_info *info, swp_entry_t *entry, unsigned long value) 392{ 393 long incdec = value? 1: -1; 394 395 entry->val = value; 396 info->swapped += incdec; 397 if ((unsigned long)(entry - info->i_direct) >= SHMEM_NR_DIRECT) { 398 struct page *page = kmap_atomic_to_page(entry); 399 set_page_private(page, page_private(page) + incdec); 400 } 401} 402 403/** 404 * shmem_swp_alloc - get the position of the swap entry for the page. 405 * @info: info structure for the inode 406 * @index: index of the page to find 407 * @sgp: check and recheck i_size? skip allocation? 408 * 409 * If the entry does not exist, allocate it. 410 */ 411static swp_entry_t *shmem_swp_alloc(struct shmem_inode_info *info, unsigned long index, enum sgp_type sgp) 412{ 413 struct inode *inode = &info->vfs_inode; 414 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 415 struct page *page = NULL; 416 swp_entry_t *entry; 417 418 if (sgp != SGP_WRITE && 419 ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) 420 return ERR_PTR(-EINVAL); 421 422 while (!(entry = shmem_swp_entry(info, index, &page))) { 423 if (sgp == SGP_READ) 424 return shmem_swp_map(ZERO_PAGE(0)); 425 /* 426 * Test used_blocks against 1 less max_blocks, since we have 1 data 427 * page (and perhaps indirect index pages) yet to allocate: 428 * a waste to allocate index if we cannot allocate data. 429 */ 430 if (sbinfo->max_blocks) { 431 if (percpu_counter_compare(&sbinfo->used_blocks, 432 sbinfo->max_blocks - 1) >= 0) 433 return ERR_PTR(-ENOSPC); 434 percpu_counter_inc(&sbinfo->used_blocks); 435 spin_lock(&inode->i_lock); 436 inode->i_blocks += BLOCKS_PER_PAGE; 437 spin_unlock(&inode->i_lock); 438 } 439 440 spin_unlock(&info->lock); 441 page = shmem_dir_alloc(mapping_gfp_mask(inode->i_mapping)); 442 spin_lock(&info->lock); 443 444 if (!page) { 445 shmem_free_blocks(inode, 1); 446 return ERR_PTR(-ENOMEM); 447 } 448 if (sgp != SGP_WRITE && 449 ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) { 450 entry = ERR_PTR(-EINVAL); 451 break; 452 } 453 if (info->next_index <= index) 454 info->next_index = index + 1; 455 } 456 if (page) { 457 /* another task gave its page, or truncated the file */ 458 shmem_free_blocks(inode, 1); 459 shmem_dir_free(page); 460 } 461 if (info->next_index <= index && !IS_ERR(entry)) 462 info->next_index = index + 1; 463 return entry; 464} 465 466/** 467 * shmem_free_swp - free some swap entries in a directory 468 * @dir: pointer to the directory 469 * @edir: pointer after last entry of the directory 470 * @punch_lock: pointer to spinlock when needed for the holepunch case 471 */ 472static int shmem_free_swp(swp_entry_t *dir, swp_entry_t *edir, 473 spinlock_t *punch_lock) 474{ 475 spinlock_t *punch_unlock = NULL; 476 swp_entry_t *ptr; 477 int freed = 0; 478 479 for (ptr = dir; ptr < edir; ptr++) { 480 if (ptr->val) { 481 if (unlikely(punch_lock)) { 482 punch_unlock = punch_lock; 483 punch_lock = NULL; 484 spin_lock(punch_unlock); 485 if (!ptr->val) 486 continue; 487 } 488 free_swap_and_cache(*ptr); 489 *ptr = (swp_entry_t){0}; 490 freed++; 491 } 492 } 493 if (punch_unlock) 494 spin_unlock(punch_unlock); 495 return freed; 496} 497 498static int shmem_map_and_free_swp(struct page *subdir, int offset, 499 int limit, struct page ***dir, spinlock_t *punch_lock) 500{ 501 swp_entry_t *ptr; 502 int freed = 0; 503 504 ptr = shmem_swp_map(subdir); 505 for (; offset < limit; offset += LATENCY_LIMIT) { 506 int size = limit - offset; 507 if (size > LATENCY_LIMIT) 508 size = LATENCY_LIMIT; 509 freed += shmem_free_swp(ptr+offset, ptr+offset+size, 510 punch_lock); 511 if (need_resched()) { 512 shmem_swp_unmap(ptr); 513 if (*dir) { 514 shmem_dir_unmap(*dir); 515 *dir = NULL; 516 } 517 cond_resched(); 518 ptr = shmem_swp_map(subdir); 519 } 520 } 521 shmem_swp_unmap(ptr); 522 return freed; 523} 524 525static void shmem_free_pages(struct list_head *next) 526{ 527 struct page *page; 528 int freed = 0; 529 530 do { 531 page = container_of(next, struct page, lru); 532 next = next->next; 533 shmem_dir_free(page); 534 freed++; 535 if (freed >= LATENCY_LIMIT) { 536 cond_resched(); 537 freed = 0; 538 } 539 } while (next); 540} 541 542static void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end) 543{ 544 struct shmem_inode_info *info = SHMEM_I(inode); 545 unsigned long idx; 546 unsigned long size; 547 unsigned long limit; 548 unsigned long stage; 549 unsigned long diroff; 550 struct page **dir; 551 struct page *topdir; 552 struct page *middir; 553 struct page *subdir; 554 swp_entry_t *ptr; 555 LIST_HEAD(pages_to_free); 556 long nr_pages_to_free = 0; 557 long nr_swaps_freed = 0; 558 int offset; 559 int freed; 560 int punch_hole; 561 spinlock_t *needs_lock; 562 spinlock_t *punch_lock; 563 unsigned long upper_limit; 564 565 inode->i_ctime = inode->i_mtime = CURRENT_TIME; 566 idx = (start + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 567 if (idx >= info->next_index) 568 return; 569 570 spin_lock(&info->lock); 571 info->flags |= SHMEM_TRUNCATE; 572 if (likely(end == (loff_t) -1)) { 573 limit = info->next_index; 574 upper_limit = SHMEM_MAX_INDEX; 575 info->next_index = idx; 576 needs_lock = NULL; 577 punch_hole = 0; 578 } else { 579 if (end + 1 >= inode->i_size) { /* we may free a little more */ 580 limit = (inode->i_size + PAGE_CACHE_SIZE - 1) >> 581 PAGE_CACHE_SHIFT; 582 upper_limit = SHMEM_MAX_INDEX; 583 } else { 584 limit = (end + 1) >> PAGE_CACHE_SHIFT; 585 upper_limit = limit; 586 } 587 needs_lock = &info->lock; 588 punch_hole = 1; 589 } 590 591 topdir = info->i_indirect; 592 if (topdir && idx <= SHMEM_NR_DIRECT && !punch_hole) { 593 info->i_indirect = NULL; 594 nr_pages_to_free++; 595 list_add(&topdir->lru, &pages_to_free); 596 } 597 spin_unlock(&info->lock); 598 599 if (info->swapped && idx < SHMEM_NR_DIRECT) { 600 ptr = info->i_direct; 601 size = limit; 602 if (size > SHMEM_NR_DIRECT) 603 size = SHMEM_NR_DIRECT; 604 nr_swaps_freed = shmem_free_swp(ptr+idx, ptr+size, needs_lock); 605 } 606 607 /* 608 * If there are no indirect blocks or we are punching a hole 609 * below indirect blocks, nothing to be done. 610 */ 611 if (!topdir || limit <= SHMEM_NR_DIRECT) 612 goto done2; 613 614 /* 615 * The truncation case has already dropped info->lock, and we're safe 616 * because i_size and next_index have already been lowered, preventing 617 * access beyond. But in the punch_hole case, we still need to take 618 * the lock when updating the swap directory, because there might be 619 * racing accesses by shmem_getpage(SGP_CACHE), shmem_unuse_inode or 620 * shmem_writepage. However, whenever we find we can remove a whole 621 * directory page (not at the misaligned start or end of the range), 622 * we first NULLify its pointer in the level above, and then have no 623 * need to take the lock when updating its contents: needs_lock and 624 * punch_lock (either pointing to info->lock or NULL) manage this. 625 */ 626 627 upper_limit -= SHMEM_NR_DIRECT; 628 limit -= SHMEM_NR_DIRECT; 629 idx = (idx > SHMEM_NR_DIRECT)? (idx - SHMEM_NR_DIRECT): 0; 630 offset = idx % ENTRIES_PER_PAGE; 631 idx -= offset; 632 633 dir = shmem_dir_map(topdir); 634 stage = ENTRIES_PER_PAGEPAGE/2; 635 if (idx < ENTRIES_PER_PAGEPAGE/2) { 636 middir = topdir; 637 diroff = idx/ENTRIES_PER_PAGE; 638 } else { 639 dir += ENTRIES_PER_PAGE/2; 640 dir += (idx - ENTRIES_PER_PAGEPAGE/2)/ENTRIES_PER_PAGEPAGE; 641 while (stage <= idx) 642 stage += ENTRIES_PER_PAGEPAGE; 643 middir = *dir; 644 if (*dir) { 645 diroff = ((idx - ENTRIES_PER_PAGEPAGE/2) % 646 ENTRIES_PER_PAGEPAGE) / ENTRIES_PER_PAGE; 647 if (!diroff && !offset && upper_limit >= stage) { 648 if (needs_lock) { 649 spin_lock(needs_lock); 650 *dir = NULL; 651 spin_unlock(needs_lock); 652 needs_lock = NULL; 653 } else 654 *dir = NULL; 655 nr_pages_to_free++; 656 list_add(&middir->lru, &pages_to_free); 657 } 658 shmem_dir_unmap(dir); 659 dir = shmem_dir_map(middir); 660 } else { 661 diroff = 0; 662 offset = 0; 663 idx = stage; 664 } 665 } 666 667 for (; idx < limit; idx += ENTRIES_PER_PAGE, diroff++) { 668 if (unlikely(idx == stage)) { 669 shmem_dir_unmap(dir); 670 dir = shmem_dir_map(topdir) + 671 ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE; 672 while (!*dir) { 673 dir++; 674 idx += ENTRIES_PER_PAGEPAGE; 675 if (idx >= limit) 676 goto done1; 677 } 678 stage = idx + ENTRIES_PER_PAGEPAGE; 679 middir = *dir; 680 if (punch_hole) 681 needs_lock = &info->lock; 682 if (upper_limit >= stage) { 683 if (needs_lock) { 684 spin_lock(needs_lock); 685 *dir = NULL; 686 spin_unlock(needs_lock); 687 needs_lock = NULL; 688 } else 689 *dir = NULL; 690 nr_pages_to_free++; 691 list_add(&middir->lru, &pages_to_free); 692 } 693 shmem_dir_unmap(dir); 694 cond_resched(); 695 dir = shmem_dir_map(middir); 696 diroff = 0; 697 } 698 punch_lock = needs_lock; 699 subdir = dir[diroff]; 700 if (subdir && !offset && upper_limit-idx >= ENTRIES_PER_PAGE) { 701 if (needs_lock) { 702 spin_lock(needs_lock); 703 dir[diroff] = NULL; 704 spin_unlock(needs_lock); 705 punch_lock = NULL; 706 } else 707 dir[diroff] = NULL; 708 nr_pages_to_free++; 709 list_add(&subdir->lru, &pages_to_free); 710 } 711 if (subdir && page_private(subdir) /* has swap entries */) { 712 size = limit - idx; 713 if (size > ENTRIES_PER_PAGE) 714 size = ENTRIES_PER_PAGE; 715 freed = shmem_map_and_free_swp(subdir, 716 offset, size, &dir, punch_lock); 717 if (!dir) 718 dir = shmem_dir_map(middir); 719 nr_swaps_freed += freed; 720 if (offset || punch_lock) { 721 spin_lock(&info->lock); 722 set_page_private(subdir, 723 page_private(subdir) - freed); 724 spin_unlock(&info->lock); 725 } else 726 BUG_ON(page_private(subdir) != freed); 727 } 728 offset = 0; 729 } 730done1: 731 shmem_dir_unmap(dir); 732done2: 733 if (inode->i_mapping->nrpages && (info->flags & SHMEM_PAGEIN)) { 734 /* 735 * Call truncate_inode_pages again: racing shmem_unuse_inode 736 * may have swizzled a page in from swap since 737 * truncate_pagecache or generic_delete_inode did it, before we 738 * lowered next_index. Also, though shmem_getpage checks 739 * i_size before adding to cache, no recheck after: so fix the 740 * narrow window there too. 741 * 742 * Recalling truncate_inode_pages_range and unmap_mapping_range 743 * every time for punch_hole (which never got a chance to clear 744 * SHMEM_PAGEIN at the start of vmtruncate_range) is expensive, 745 * yet hardly ever necessary: try to optimize them out later. 746 */ 747 truncate_inode_pages_range(inode->i_mapping, start, end); 748 if (punch_hole) 749 unmap_mapping_range(inode->i_mapping, start, 750 end - start, 1); 751 } 752 753 spin_lock(&info->lock); 754 info->flags &= ~SHMEM_TRUNCATE; 755 info->swapped -= nr_swaps_freed; 756 if (nr_pages_to_free) 757 shmem_free_blocks(inode, nr_pages_to_free); 758 shmem_recalc_inode(inode); 759 spin_unlock(&info->lock); 760 761 /* 762 * Empty swap vector directory pages to be freed? 763 */ 764 if (!list_empty(&pages_to_free)) { 765 pages_to_free.prev->next = NULL; 766 shmem_free_pages(pages_to_free.next); 767 } 768} 769 770static int shmem_notify_change(struct dentry *dentry, struct iattr *attr) 771{ 772 struct inode *inode = dentry->d_inode; 773 loff_t newsize = attr->ia_size; 774 int error; 775 776 error = inode_change_ok(inode, attr); 777 if (error) 778 return error; 779 780 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE) 781 && newsize != inode->i_size) { 782 struct page *page = NULL; 783 784 if (newsize < inode->i_size) { 785 /* 786 * If truncating down to a partial page, then 787 * if that page is already allocated, hold it 788 * in memory until the truncation is over, so 789 * truncate_partial_page cannot miss it were 790 * it assigned to swap. 791 */ 792 if (newsize & (PAGE_CACHE_SIZE-1)) { 793 (void) shmem_getpage(inode, 794 newsize >> PAGE_CACHE_SHIFT, 795 &page, SGP_READ, NULL); 796 if (page) 797 unlock_page(page); 798 } 799 /* 800 * Reset SHMEM_PAGEIN flag so that shmem_truncate can 801 * detect if any pages might have been added to cache 802 * after truncate_inode_pages. But we needn't bother 803 * if it's being fully truncated to zero-length: the 804 * nrpages check is efficient enough in that case. 805 */ 806 if (newsize) { 807 struct shmem_inode_info *info = SHMEM_I(inode); 808 spin_lock(&info->lock); 809 info->flags &= ~SHMEM_PAGEIN; 810 spin_unlock(&info->lock); 811 } 812 } 813 814 /* XXX(truncate): truncate_setsize should be called last */ 815 truncate_setsize(inode, newsize); 816 if (page) 817 page_cache_release(page); 818 shmem_truncate_range(inode, newsize, (loff_t)-1); 819 } 820 821 setattr_copy(inode, attr); 822#ifdef CONFIG_TMPFS_POSIX_ACL 823 if (attr->ia_valid & ATTR_MODE) 824 error = generic_acl_chmod(inode); 825#endif 826 return error; 827} 828 829static void shmem_evict_inode(struct inode *inode) 830{ 831 struct shmem_inode_info *info = SHMEM_I(inode); 832 struct shmem_xattr *xattr, *nxattr; 833 834 if (inode->i_mapping->a_ops == &shmem_aops) { 835 truncate_inode_pages(inode->i_mapping, 0); 836 shmem_unacct_size(info->flags, inode->i_size); 837 inode->i_size = 0; 838 shmem_truncate_range(inode, 0, (loff_t)-1); 839 if (!list_empty(&info->swaplist)) { 840 mutex_lock(&shmem_swaplist_mutex); 841 list_del_init(&info->swaplist); 842 mutex_unlock(&shmem_swaplist_mutex); 843 } 844 } 845 846 list_for_each_entry_safe(xattr, nxattr, &info->xattr_list, list) { 847 kfree(xattr->name); 848 kfree(xattr); 849 } 850 BUG_ON(inode->i_blocks); 851 shmem_free_inode(inode->i_sb); 852 end_writeback(inode); 853} 854 855static inline int shmem_find_swp(swp_entry_t entry, swp_entry_t *dir, swp_entry_t *edir) 856{ 857 swp_entry_t *ptr; 858 859 for (ptr = dir; ptr < edir; ptr++) { 860 if (ptr->val == entry.val) 861 return ptr - dir; 862 } 863 return -1; 864} 865 866static int shmem_unuse_inode(struct shmem_inode_info *info, swp_entry_t entry, struct page *page) 867{ 868 struct address_space *mapping; 869 unsigned long idx; 870 unsigned long size; 871 unsigned long limit; 872 unsigned long stage; 873 struct page **dir; 874 struct page *subdir; 875 swp_entry_t *ptr; 876 int offset; 877 int error; 878 879 idx = 0; 880 ptr = info->i_direct; 881 spin_lock(&info->lock); 882 if (!info->swapped) { 883 list_del_init(&info->swaplist); 884 goto lost2; 885 } 886 limit = info->next_index; 887 size = limit; 888 if (size > SHMEM_NR_DIRECT) 889 size = SHMEM_NR_DIRECT; 890 offset = shmem_find_swp(entry, ptr, ptr+size); 891 if (offset >= 0) { 892 shmem_swp_balance_unmap(); 893 goto found; 894 } 895 if (!info->i_indirect) 896 goto lost2; 897 898 dir = shmem_dir_map(info->i_indirect); 899 stage = SHMEM_NR_DIRECT + ENTRIES_PER_PAGEPAGE/2; 900 901 for (idx = SHMEM_NR_DIRECT; idx < limit; idx += ENTRIES_PER_PAGE, dir++) { 902 if (unlikely(idx == stage)) { 903 shmem_dir_unmap(dir-1); 904 if (cond_resched_lock(&info->lock)) { 905 /* check it has not been truncated */ 906 if (limit > info->next_index) { 907 limit = info->next_index; 908 if (idx >= limit) 909 goto lost2; 910 } 911 } 912 dir = shmem_dir_map(info->i_indirect) + 913 ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE; 914 while (!*dir) { 915 dir++; 916 idx += ENTRIES_PER_PAGEPAGE; 917 if (idx >= limit) 918 goto lost1; 919 } 920 stage = idx + ENTRIES_PER_PAGEPAGE; 921 subdir = *dir; 922 shmem_dir_unmap(dir); 923 dir = shmem_dir_map(subdir); 924 } 925 subdir = *dir; 926 if (subdir && page_private(subdir)) { 927 ptr = shmem_swp_map(subdir); 928 size = limit - idx; 929 if (size > ENTRIES_PER_PAGE) 930 size = ENTRIES_PER_PAGE; 931 offset = shmem_find_swp(entry, ptr, ptr+size); 932 shmem_swp_unmap(ptr); 933 if (offset >= 0) { 934 shmem_dir_unmap(dir); 935 ptr = shmem_swp_map(subdir); 936 goto found; 937 } 938 } 939 } 940lost1: 941 shmem_dir_unmap(dir-1); 942lost2: 943 spin_unlock(&info->lock); 944 return 0; 945found: 946 idx += offset; 947 ptr += offset; 948 949 /* 950 * Move _head_ to start search for next from here. 951 * But be careful: shmem_evict_inode checks list_empty without taking 952 * mutex, and there's an instant in list_move_tail when info->swaplist 953 * would appear empty, if it were the only one on shmem_swaplist. We 954 * could avoid doing it if inode NULL; or use this minor optimization. 955 */ 956 if (shmem_swaplist.next != &info->swaplist) 957 list_move_tail(&shmem_swaplist, &info->swaplist); 958 959 /* 960 * We rely on shmem_swaplist_mutex, not only to protect the swaplist, 961 * but also to hold up shmem_evict_inode(): so inode cannot be freed 962 * beneath us (pagelock doesn't help until the page is in pagecache). 963 */ 964 mapping = info->vfs_inode.i_mapping; 965 error = add_to_page_cache_locked(page, mapping, idx, GFP_NOWAIT); 966 /* which does mem_cgroup_uncharge_cache_page on error */ 967 968 if (error == -EEXIST) { 969 struct page *filepage = find_get_page(mapping, idx); 970 error = 1; 971 if (filepage) { 972 /* 973 * There might be a more uptodate page coming down 974 * from a stacked writepage: forget our swappage if so. 975 */ 976 if (PageUptodate(filepage)) 977 error = 0; 978 page_cache_release(filepage); 979 } 980 } 981 if (!error) { 982 delete_from_swap_cache(page); 983 set_page_dirty(page); 984 info->flags |= SHMEM_PAGEIN; 985 shmem_swp_set(info, ptr, 0); 986 swap_free(entry); 987 error = 1; /* not an error, but entry was found */ 988 } 989 shmem_swp_unmap(ptr); 990 spin_unlock(&info->lock); 991 return error; 992} 993 994/* 995 * shmem_unuse() search for an eventually swapped out shmem page. 996 */ 997int shmem_unuse(swp_entry_t entry, struct page *page) 998{ 999 struct list_head *p, *next; 1000 struct shmem_inode_info *info; 1001 int found = 0; 1002 int error; 1003 1004 /* 1005 * Charge page using GFP_KERNEL while we can wait, before taking 1006 * the shmem_swaplist_mutex which might hold up shmem_writepage(). 1007 * Charged back to the user (not to caller) when swap account is used. 1008 * add_to_page_cache() will be called with GFP_NOWAIT. 1009 */ 1010 error = mem_cgroup_cache_charge(page, current->mm, GFP_KERNEL); 1011 if (error) 1012 goto out; 1013 /* 1014 * Try to preload while we can wait, to not make a habit of 1015 * draining atomic reserves; but don't latch on to this cpu, 1016 * it's okay if sometimes we get rescheduled after this. 1017 */ 1018 error = radix_tree_preload(GFP_KERNEL); 1019 if (error) 1020 goto uncharge; 1021 radix_tree_preload_end(); 1022 1023 mutex_lock(&shmem_swaplist_mutex); 1024 list_for_each_safe(p, next, &shmem_swaplist) { 1025 info = list_entry(p, struct shmem_inode_info, swaplist); 1026 found = shmem_unuse_inode(info, entry, page); 1027 cond_resched(); 1028 if (found) 1029 break; 1030 } 1031 mutex_unlock(&shmem_swaplist_mutex); 1032 1033uncharge: 1034 if (!found) 1035 mem_cgroup_uncharge_cache_page(page); 1036 if (found < 0) 1037 error = found; 1038out: 1039 unlock_page(page); 1040 page_cache_release(page); 1041 return error; 1042} 1043 1044/* 1045 * Move the page from the page cache to the swap cache. 1046 */ 1047static int shmem_writepage(struct page *page, struct writeback_control *wbc) 1048{ 1049 struct shmem_inode_info *info; 1050 swp_entry_t *entry, swap; 1051 struct address_space *mapping; 1052 unsigned long index; 1053 struct inode *inode; 1054 1055 BUG_ON(!PageLocked(page)); 1056 mapping = page->mapping; 1057 index = page->index; 1058 inode = mapping->host; 1059 info = SHMEM_I(inode); 1060 if (info->flags & VM_LOCKED) 1061 goto redirty; 1062 if (!total_swap_pages) 1063 goto redirty; 1064 1065 /* 1066 * shmem_backing_dev_info's capabilities prevent regular writeback or 1067 * sync from ever calling shmem_writepage; but a stacking filesystem 1068 * may use the ->writepage of its underlying filesystem, in which case 1069 * tmpfs should write out to swap only in response to memory pressure, 1070 * and not for the writeback threads or sync. However, in those cases, 1071 * we do still want to check if there's a redundant swappage to be 1072 * discarded. 1073 */ 1074 if (wbc->for_reclaim) 1075 swap = get_swap_page(); 1076 else 1077 swap.val = 0; 1078 1079 /* 1080 * Add inode to shmem_unuse()'s list of swapped-out inodes, 1081 * if it's not already there. Do it now because we cannot take 1082 * mutex while holding spinlock, and must do so before the page 1083 * is moved to swap cache, when its pagelock no longer protects 1084 * the inode from eviction. But don't unlock the mutex until 1085 * we've taken the spinlock, because shmem_unuse_inode() will 1086 * prune a !swapped inode from the swaplist under both locks. 1087 */ 1088 if (swap.val) { 1089 mutex_lock(&shmem_swaplist_mutex); 1090 if (list_empty(&info->swaplist)) 1091 list_add_tail(&info->swaplist, &shmem_swaplist); 1092 } 1093 1094 spin_lock(&info->lock); 1095 if (swap.val) 1096 mutex_unlock(&shmem_swaplist_mutex); 1097 1098 if (index >= info->next_index) { 1099 BUG_ON(!(info->flags & SHMEM_TRUNCATE)); 1100 goto unlock; 1101 } 1102 entry = shmem_swp_entry(info, index, NULL); 1103 if (entry->val) { 1104 /* 1105 * The more uptodate page coming down from a stacked 1106 * writepage should replace our old swappage. 1107 */ 1108 free_swap_and_cache(*entry); 1109 shmem_swp_set(info, entry, 0); 1110 } 1111 shmem_recalc_inode(inode); 1112 1113 if (swap.val && add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) { 1114 delete_from_page_cache(page); 1115 shmem_swp_set(info, entry, swap.val); 1116 shmem_swp_unmap(entry); 1117 spin_unlock(&info->lock); 1118 swap_shmem_alloc(swap); 1119 BUG_ON(page_mapped(page)); 1120 swap_writepage(page, wbc); 1121 return 0; 1122 } 1123 1124 shmem_swp_unmap(entry); 1125unlock: 1126 spin_unlock(&info->lock); 1127 /* 1128 * add_to_swap_cache() doesn't return -EEXIST, so we can safely 1129 * clear SWAP_HAS_CACHE flag. 1130 */ 1131 swapcache_free(swap, NULL); 1132redirty: 1133 set_page_dirty(page); 1134 if (wbc->for_reclaim) 1135 return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */ 1136 unlock_page(page); 1137 return 0; 1138} 1139 1140#ifdef CONFIG_NUMA 1141#ifdef CONFIG_TMPFS 1142static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1143{ 1144 char buffer[64]; 1145 1146 if (!mpol || mpol->mode == MPOL_DEFAULT) 1147 return; /* show nothing */ 1148 1149 mpol_to_str(buffer, sizeof(buffer), mpol, 1); 1150 1151 seq_printf(seq, ",mpol=%s", buffer); 1152} 1153 1154static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1155{ 1156 struct mempolicy *mpol = NULL; 1157 if (sbinfo->mpol) { 1158 spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */ 1159 mpol = sbinfo->mpol; 1160 mpol_get(mpol); 1161 spin_unlock(&sbinfo->stat_lock); 1162 } 1163 return mpol; 1164} 1165#endif /* CONFIG_TMPFS */ 1166 1167static struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp, 1168 struct shmem_inode_info *info, unsigned long idx) 1169{ 1170 struct mempolicy mpol, *spol; 1171 struct vm_area_struct pvma; 1172 struct page *page; 1173 1174 spol = mpol_cond_copy(&mpol, 1175 mpol_shared_policy_lookup(&info->policy, idx)); 1176 1177 /* Create a pseudo vma that just contains the policy */ 1178 pvma.vm_start = 0; 1179 pvma.vm_pgoff = idx; 1180 pvma.vm_ops = NULL; 1181 pvma.vm_policy = spol; 1182 page = swapin_readahead(entry, gfp, &pvma, 0); 1183 return page; 1184} 1185 1186static struct page *shmem_alloc_page(gfp_t gfp, 1187 struct shmem_inode_info *info, unsigned long idx) 1188{ 1189 struct vm_area_struct pvma; 1190 1191 /* Create a pseudo vma that just contains the policy */ 1192 pvma.vm_start = 0; 1193 pvma.vm_pgoff = idx; 1194 pvma.vm_ops = NULL; 1195 pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx); 1196 1197 /* 1198 * alloc_page_vma() will drop the shared policy reference 1199 */ 1200 return alloc_page_vma(gfp, &pvma, 0); 1201} 1202#else /* !CONFIG_NUMA */ 1203#ifdef CONFIG_TMPFS 1204static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *p) 1205{ 1206} 1207#endif /* CONFIG_TMPFS */ 1208 1209static inline struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp, 1210 struct shmem_inode_info *info, unsigned long idx) 1211{ 1212 return swapin_readahead(entry, gfp, NULL, 0); 1213} 1214 1215static inline struct page *shmem_alloc_page(gfp_t gfp, 1216 struct shmem_inode_info *info, unsigned long idx) 1217{ 1218 return alloc_page(gfp); 1219} 1220#endif /* CONFIG_NUMA */ 1221 1222#if !defined(CONFIG_NUMA) || !defined(CONFIG_TMPFS) 1223static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1224{ 1225 return NULL; 1226} 1227#endif 1228 1229/* 1230 * shmem_getpage - either get the page from swap or allocate a new one 1231 * 1232 * If we allocate a new one we do not mark it dirty. That's up to the 1233 * vm. If we swap it in we mark it dirty since we also free the swap 1234 * entry since a page cannot live in both the swap and page cache 1235 */ 1236static int shmem_getpage(struct inode *inode, unsigned long idx, 1237 struct page **pagep, enum sgp_type sgp, int *type) 1238{ 1239 struct address_space *mapping = inode->i_mapping; 1240 struct shmem_inode_info *info = SHMEM_I(inode); 1241 struct shmem_sb_info *sbinfo; 1242 struct page *filepage = *pagep; 1243 struct page *swappage; 1244 struct page *prealloc_page = NULL; 1245 swp_entry_t *entry; 1246 swp_entry_t swap; 1247 gfp_t gfp; 1248 int error; 1249 1250 if (idx >= SHMEM_MAX_INDEX) 1251 return -EFBIG; 1252 1253 if (type) 1254 *type = 0; 1255 1256 /* 1257 * Normally, filepage is NULL on entry, and either found 1258 * uptodate immediately, or allocated and zeroed, or read 1259 * in under swappage, which is then assigned to filepage. 1260 * But shmem_readpage (required for splice) passes in a locked 1261 * filepage, which may be found not uptodate by other callers 1262 * too, and may need to be copied from the swappage read in. 1263 */ 1264repeat: 1265 if (!filepage) 1266 filepage = find_lock_page(mapping, idx); 1267 if (filepage && PageUptodate(filepage)) 1268 goto done; 1269 gfp = mapping_gfp_mask(mapping); 1270 if (!filepage) { 1271 /* 1272 * Try to preload while we can wait, to not make a habit of 1273 * draining atomic reserves; but don't latch on to this cpu. 1274 */ 1275 error = radix_tree_preload(gfp & ~__GFP_HIGHMEM); 1276 if (error) 1277 goto failed; 1278 radix_tree_preload_end(); 1279 if (sgp != SGP_READ && !prealloc_page) { 1280 /* We don't care if this fails */ 1281 prealloc_page = shmem_alloc_page(gfp, info, idx); 1282 if (prealloc_page) { 1283 if (mem_cgroup_cache_charge(prealloc_page, 1284 current->mm, GFP_KERNEL)) { 1285 page_cache_release(prealloc_page); 1286 prealloc_page = NULL; 1287 } 1288 } 1289 } 1290 } 1291 error = 0; 1292 1293 spin_lock(&info->lock); 1294 shmem_recalc_inode(inode); 1295 entry = shmem_swp_alloc(info, idx, sgp); 1296 if (IS_ERR(entry)) { 1297 spin_unlock(&info->lock); 1298 error = PTR_ERR(entry); 1299 goto failed; 1300 } 1301 swap = *entry; 1302 1303 if (swap.val) { 1304 /* Look it up and read it in.. */ 1305 swappage = lookup_swap_cache(swap); 1306 if (!swappage) { 1307 shmem_swp_unmap(entry); 1308 /* here we actually do the io */ 1309 if (type && !(*type & VM_FAULT_MAJOR)) { 1310 __count_vm_event(PGMAJFAULT); 1311 *type |= VM_FAULT_MAJOR; 1312 } 1313 spin_unlock(&info->lock); 1314 swappage = shmem_swapin(swap, gfp, info, idx); 1315 if (!swappage) { 1316 spin_lock(&info->lock); 1317 entry = shmem_swp_alloc(info, idx, sgp); 1318 if (IS_ERR(entry)) 1319 error = PTR_ERR(entry); 1320 else { 1321 if (entry->val == swap.val) 1322 error = -ENOMEM; 1323 shmem_swp_unmap(entry); 1324 } 1325 spin_unlock(&info->lock); 1326 if (error) 1327 goto failed; 1328 goto repeat; 1329 } 1330 wait_on_page_locked(swappage); 1331 page_cache_release(swappage); 1332 goto repeat; 1333 } 1334 1335 /* We have to do this with page locked to prevent races */ 1336 if (!trylock_page(swappage)) { 1337 shmem_swp_unmap(entry); 1338 spin_unlock(&info->lock); 1339 wait_on_page_locked(swappage); 1340 page_cache_release(swappage); 1341 goto repeat; 1342 } 1343 if (PageWriteback(swappage)) { 1344 shmem_swp_unmap(entry); 1345 spin_unlock(&info->lock); 1346 wait_on_page_writeback(swappage); 1347 unlock_page(swappage); 1348 page_cache_release(swappage); 1349 goto repeat; 1350 } 1351 if (!PageUptodate(swappage)) { 1352 shmem_swp_unmap(entry); 1353 spin_unlock(&info->lock); 1354 unlock_page(swappage); 1355 page_cache_release(swappage); 1356 error = -EIO; 1357 goto failed; 1358 } 1359 1360 if (filepage) { 1361 shmem_swp_set(info, entry, 0); 1362 shmem_swp_unmap(entry); 1363 delete_from_swap_cache(swappage); 1364 spin_unlock(&info->lock); 1365 copy_highpage(filepage, swappage); 1366 unlock_page(swappage); 1367 page_cache_release(swappage); 1368 flush_dcache_page(filepage); 1369 SetPageUptodate(filepage); 1370 set_page_dirty(filepage); 1371 swap_free(swap); 1372 } else if (!(error = add_to_page_cache_locked(swappage, mapping, 1373 idx, GFP_NOWAIT))) { 1374 info->flags |= SHMEM_PAGEIN; 1375 shmem_swp_set(info, entry, 0); 1376 shmem_swp_unmap(entry); 1377 delete_from_swap_cache(swappage); 1378 spin_unlock(&info->lock); 1379 filepage = swappage; 1380 set_page_dirty(filepage); 1381 swap_free(swap); 1382 } else { 1383 shmem_swp_unmap(entry); 1384 spin_unlock(&info->lock); 1385 if (error == -ENOMEM) { 1386 /* 1387 * reclaim from proper memory cgroup and 1388 * call memcg's OOM if needed. 1389 */ 1390 error = mem_cgroup_shmem_charge_fallback( 1391 swappage, 1392 current->mm, 1393 gfp); 1394 if (error) { 1395 unlock_page(swappage); 1396 page_cache_release(swappage); 1397 goto failed; 1398 } 1399 } 1400 unlock_page(swappage); 1401 page_cache_release(swappage); 1402 goto repeat; 1403 } 1404 } else if (sgp == SGP_READ && !filepage) { 1405 shmem_swp_unmap(entry); 1406 filepage = find_get_page(mapping, idx); 1407 if (filepage && 1408 (!PageUptodate(filepage) || !trylock_page(filepage))) { 1409 spin_unlock(&info->lock); 1410 wait_on_page_locked(filepage); 1411 page_cache_release(filepage); 1412 filepage = NULL; 1413 goto repeat; 1414 } 1415 spin_unlock(&info->lock); 1416 } else { 1417 shmem_swp_unmap(entry); 1418 sbinfo = SHMEM_SB(inode->i_sb); 1419 if (sbinfo->max_blocks) { 1420 if (percpu_counter_compare(&sbinfo->used_blocks, 1421 sbinfo->max_blocks) >= 0 || 1422 shmem_acct_block(info->flags)) 1423 goto nospace; 1424 percpu_counter_inc(&sbinfo->used_blocks); 1425 spin_lock(&inode->i_lock); 1426 inode->i_blocks += BLOCKS_PER_PAGE; 1427 spin_unlock(&inode->i_lock); 1428 } else if (shmem_acct_block(info->flags)) 1429 goto nospace; 1430 1431 if (!filepage) { 1432 int ret; 1433 1434 if (!prealloc_page) { 1435 spin_unlock(&info->lock); 1436 filepage = shmem_alloc_page(gfp, info, idx); 1437 if (!filepage) { 1438 shmem_unacct_blocks(info->flags, 1); 1439 shmem_free_blocks(inode, 1); 1440 error = -ENOMEM; 1441 goto failed; 1442 } 1443 SetPageSwapBacked(filepage); 1444 1445 /* 1446 * Precharge page while we can wait, compensate 1447 * after 1448 */ 1449 error = mem_cgroup_cache_charge(filepage, 1450 current->mm, GFP_KERNEL); 1451 if (error) { 1452 page_cache_release(filepage); 1453 shmem_unacct_blocks(info->flags, 1); 1454 shmem_free_blocks(inode, 1); 1455 filepage = NULL; 1456 goto failed; 1457 } 1458 1459 spin_lock(&info->lock); 1460 } else { 1461 filepage = prealloc_page; 1462 prealloc_page = NULL; 1463 SetPageSwapBacked(filepage); 1464 } 1465 1466 entry = shmem_swp_alloc(info, idx, sgp); 1467 if (IS_ERR(entry)) 1468 error = PTR_ERR(entry); 1469 else { 1470 swap = *entry; 1471 shmem_swp_unmap(entry); 1472 } 1473 ret = error || swap.val; 1474 if (ret) 1475 mem_cgroup_uncharge_cache_page(filepage); 1476 else 1477 ret = add_to_page_cache_lru(filepage, mapping, 1478 idx, GFP_NOWAIT); 1479 /* 1480 * At add_to_page_cache_lru() failure, uncharge will 1481 * be done automatically. 1482 */ 1483 if (ret) { 1484 spin_unlock(&info->lock); 1485 page_cache_release(filepage); 1486 shmem_unacct_blocks(info->flags, 1); 1487 shmem_free_blocks(inode, 1); 1488 filepage = NULL; 1489 if (error) 1490 goto failed; 1491 goto repeat; 1492 } 1493 info->flags |= SHMEM_PAGEIN; 1494 } 1495 1496 info->alloced++; 1497 spin_unlock(&info->lock); 1498 clear_highpage(filepage); 1499 flush_dcache_page(filepage); 1500 SetPageUptodate(filepage); 1501 if (sgp == SGP_DIRTY) 1502 set_page_dirty(filepage); 1503 } 1504done: 1505 *pagep = filepage; 1506 error = 0; 1507 goto out; 1508 1509nospace: 1510 /* 1511 * Perhaps the page was brought in from swap between find_lock_page 1512 * and taking info->lock? We allow for that at add_to_page_cache_lru, 1513 * but must also avoid reporting a spurious ENOSPC while working on a 1514 * full tmpfs. (When filepage has been passed in to shmem_getpage, it 1515 * is already in page cache, which prevents this race from occurring.) 1516 */ 1517 if (!filepage) { 1518 struct page *page = find_get_page(mapping, idx); 1519 if (page) { 1520 spin_unlock(&info->lock); 1521 page_cache_release(page); 1522 goto repeat; 1523 } 1524 } 1525 spin_unlock(&info->lock); 1526 error = -ENOSPC; 1527failed: 1528 if (*pagep != filepage) { 1529 unlock_page(filepage); 1530 page_cache_release(filepage); 1531 } 1532out: 1533 if (prealloc_page) { 1534 mem_cgroup_uncharge_cache_page(prealloc_page); 1535 page_cache_release(prealloc_page); 1536 } 1537 return error; 1538} 1539 1540static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1541{ 1542 struct inode *inode = vma->vm_file->f_path.dentry->d_inode; 1543 int error; 1544 int ret; 1545 1546 if (((loff_t)vmf->pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode)) 1547 return VM_FAULT_SIGBUS; 1548 1549 error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret); 1550 if (error) 1551 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS); 1552 1553 return ret | VM_FAULT_LOCKED; 1554} 1555 1556#ifdef CONFIG_NUMA 1557static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *new) 1558{ 1559 struct inode *i = vma->vm_file->f_path.dentry->d_inode; 1560 return mpol_set_shared_policy(&SHMEM_I(i)->policy, vma, new); 1561} 1562 1563static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, 1564 unsigned long addr) 1565{ 1566 struct inode *i = vma->vm_file->f_path.dentry->d_inode; 1567 unsigned long idx; 1568 1569 idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 1570 return mpol_shared_policy_lookup(&SHMEM_I(i)->policy, idx); 1571} 1572#endif 1573 1574int shmem_lock(struct file *file, int lock, struct user_struct *user) 1575{ 1576 struct inode *inode = file->f_path.dentry->d_inode; 1577 struct shmem_inode_info *info = SHMEM_I(inode); 1578 int retval = -ENOMEM; 1579 1580 spin_lock(&info->lock); 1581 if (lock && !(info->flags & VM_LOCKED)) { 1582 if (!user_shm_lock(inode->i_size, user)) 1583 goto out_nomem; 1584 info->flags |= VM_LOCKED; 1585 mapping_set_unevictable(file->f_mapping); 1586 } 1587 if (!lock && (info->flags & VM_LOCKED) && user) { 1588 user_shm_unlock(inode->i_size, user); 1589 info->flags &= ~VM_LOCKED; 1590 mapping_clear_unevictable(file->f_mapping); 1591 scan_mapping_unevictable_pages(file->f_mapping); 1592 } 1593 retval = 0; 1594 1595out_nomem: 1596 spin_unlock(&info->lock); 1597 return retval; 1598} 1599 1600static int shmem_mmap(struct file *file, struct vm_area_struct *vma) 1601{ 1602 file_accessed(file); 1603 vma->vm_ops = &shmem_vm_ops; 1604 vma->vm_flags |= VM_CAN_NONLINEAR; 1605 return 0; 1606} 1607 1608static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir, 1609 int mode, dev_t dev, unsigned long flags) 1610{ 1611 struct inode *inode; 1612 struct shmem_inode_info *info; 1613 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 1614 1615 if (shmem_reserve_inode(sb)) 1616 return NULL; 1617 1618 inode = new_inode(sb); 1619 if (inode) { 1620 inode->i_ino = get_next_ino(); 1621 inode_init_owner(inode, dir, mode); 1622 inode->i_blocks = 0; 1623 inode->i_mapping->backing_dev_info = &shmem_backing_dev_info; 1624 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME; 1625 inode->i_generation = get_seconds(); 1626 info = SHMEM_I(inode); 1627 memset(info, 0, (char *)inode - (char *)info); 1628 spin_lock_init(&info->lock); 1629 info->flags = flags & VM_NORESERVE; 1630 INIT_LIST_HEAD(&info->swaplist); 1631 INIT_LIST_HEAD(&info->xattr_list); 1632 cache_no_acl(inode); 1633 1634 switch (mode & S_IFMT) { 1635 default: 1636 inode->i_op = &shmem_special_inode_operations; 1637 init_special_inode(inode, mode, dev); 1638 break; 1639 case S_IFREG: 1640 inode->i_mapping->a_ops = &shmem_aops; 1641 inode->i_op = &shmem_inode_operations; 1642 inode->i_fop = &shmem_file_operations; 1643 mpol_shared_policy_init(&info->policy, 1644 shmem_get_sbmpol(sbinfo)); 1645 break; 1646 case S_IFDIR: 1647 inc_nlink(inode); 1648 /* Some things misbehave if size == 0 on a directory */ 1649 inode->i_size = 2 * BOGO_DIRENT_SIZE; 1650 inode->i_op = &shmem_dir_inode_operations; 1651 inode->i_fop = &simple_dir_operations; 1652 break; 1653 case S_IFLNK: 1654 /* 1655 * Must not load anything in the rbtree, 1656 * mpol_free_shared_policy will not be called. 1657 */ 1658 mpol_shared_policy_init(&info->policy, NULL); 1659 break; 1660 } 1661 } else 1662 shmem_free_inode(sb); 1663 return inode; 1664} 1665 1666#ifdef CONFIG_TMPFS 1667static const struct inode_operations shmem_symlink_inode_operations; 1668static const struct inode_operations shmem_symlink_inline_operations; 1669 1670/* 1671 * Normally tmpfs avoids the use of shmem_readpage and shmem_write_begin; 1672 * but providing them allows a tmpfs file to be used for splice, sendfile, and 1673 * below the loop driver, in the generic fashion that many filesystems support. 1674 */ 1675static int shmem_readpage(struct file *file, struct page *page) 1676{ 1677 struct inode *inode = page->mapping->host; 1678 int error = shmem_getpage(inode, page->index, &page, SGP_CACHE, NULL); 1679 unlock_page(page); 1680 return error; 1681} 1682 1683static int 1684shmem_write_begin(struct file *file, struct address_space *mapping, 1685 loff_t pos, unsigned len, unsigned flags, 1686 struct page **pagep, void **fsdata) 1687{ 1688 struct inode *inode = mapping->host; 1689 pgoff_t index = pos >> PAGE_CACHE_SHIFT; 1690 *pagep = NULL; 1691 return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL); 1692} 1693 1694static int 1695shmem_write_end(struct file *file, struct address_space *mapping, 1696 loff_t pos, unsigned len, unsigned copied, 1697 struct page *page, void *fsdata) 1698{ 1699 struct inode *inode = mapping->host; 1700 1701 if (pos + copied > inode->i_size) 1702 i_size_write(inode, pos + copied); 1703 1704 set_page_dirty(page); 1705 unlock_page(page); 1706 page_cache_release(page); 1707 1708 return copied; 1709} 1710 1711static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor) 1712{ 1713 struct inode *inode = filp->f_path.dentry->d_inode; 1714 struct address_space *mapping = inode->i_mapping; 1715 unsigned long index, offset; 1716 enum sgp_type sgp = SGP_READ; 1717 1718 /* 1719 * Might this read be for a stacking filesystem? Then when reading 1720 * holes of a sparse file, we actually need to allocate those pages, 1721 * and even mark them dirty, so it cannot exceed the max_blocks limit. 1722 */ 1723 if (segment_eq(get_fs(), KERNEL_DS)) 1724 sgp = SGP_DIRTY; 1725 1726 index = *ppos >> PAGE_CACHE_SHIFT; 1727 offset = *ppos & ~PAGE_CACHE_MASK; 1728 1729 for (;;) { 1730 struct page *page = NULL; 1731 unsigned long end_index, nr, ret; 1732 loff_t i_size = i_size_read(inode); 1733 1734 end_index = i_size >> PAGE_CACHE_SHIFT; 1735 if (index > end_index) 1736 break; 1737 if (index == end_index) { 1738 nr = i_size & ~PAGE_CACHE_MASK; 1739 if (nr <= offset) 1740 break; 1741 } 1742 1743 desc->error = shmem_getpage(inode, index, &page, sgp, NULL); 1744 if (desc->error) { 1745 if (desc->error == -EINVAL) 1746 desc->error = 0; 1747 break; 1748 } 1749 if (page) 1750 unlock_page(page); 1751 1752 /* 1753 * We must evaluate after, since reads (unlike writes) 1754 * are called without i_mutex protection against truncate 1755 */ 1756 nr = PAGE_CACHE_SIZE; 1757 i_size = i_size_read(inode); 1758 end_index = i_size >> PAGE_CACHE_SHIFT; 1759 if (index == end_index) { 1760 nr = i_size & ~PAGE_CACHE_MASK; 1761 if (nr <= offset) { 1762 if (page) 1763 page_cache_release(page); 1764 break; 1765 } 1766 } 1767 nr -= offset; 1768 1769 if (page) { 1770 /* 1771 * If users can be writing to this page using arbitrary 1772 * virtual addresses, take care about potential aliasing 1773 * before reading the page on the kernel side. 1774 */ 1775 if (mapping_writably_mapped(mapping)) 1776 flush_dcache_page(page); 1777 /* 1778 * Mark the page accessed if we read the beginning. 1779 */ 1780 if (!offset) 1781 mark_page_accessed(page); 1782 } else { 1783 page = ZERO_PAGE(0); 1784 page_cache_get(page); 1785 } 1786 1787 /* 1788 * Ok, we have the page, and it's up-to-date, so 1789 * now we can copy it to user space... 1790 * 1791 * The actor routine returns how many bytes were actually used.. 1792 * NOTE! This may not be the same as how much of a user buffer 1793 * we filled up (we may be padding etc), so we can only update 1794 * "pos" here (the actor routine has to update the user buffer 1795 * pointers and the remaining count). 1796 */ 1797 ret = actor(desc, page, offset, nr); 1798 offset += ret; 1799 index += offset >> PAGE_CACHE_SHIFT; 1800 offset &= ~PAGE_CACHE_MASK; 1801 1802 page_cache_release(page); 1803 if (ret != nr || !desc->count) 1804 break; 1805 1806 cond_resched(); 1807 } 1808 1809 *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset; 1810 file_accessed(filp); 1811} 1812 1813static ssize_t shmem_file_aio_read(struct kiocb *iocb, 1814 const struct iovec *iov, unsigned long nr_segs, loff_t pos) 1815{ 1816 struct file *filp = iocb->ki_filp; 1817 ssize_t retval; 1818 unsigned long seg; 1819 size_t count; 1820 loff_t *ppos = &iocb->ki_pos; 1821 1822 retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE); 1823 if (retval) 1824 return retval; 1825 1826 for (seg = 0; seg < nr_segs; seg++) { 1827 read_descriptor_t desc; 1828 1829 desc.written = 0; 1830 desc.arg.buf = iov[seg].iov_base; 1831 desc.count = iov[seg].iov_len; 1832 if (desc.count == 0) 1833 continue; 1834 desc.error = 0; 1835 do_shmem_file_read(filp, ppos, &desc, file_read_actor); 1836 retval += desc.written; 1837 if (desc.error) { 1838 retval = retval ?: desc.error; 1839 break; 1840 } 1841 if (desc.count > 0) 1842 break; 1843 } 1844 return retval; 1845} 1846 1847static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf) 1848{ 1849 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb); 1850 1851 buf->f_type = TMPFS_MAGIC; 1852 buf->f_bsize = PAGE_CACHE_SIZE; 1853 buf->f_namelen = NAME_MAX; 1854 if (sbinfo->max_blocks) { 1855 buf->f_blocks = sbinfo->max_blocks; 1856 buf->f_bavail = buf->f_bfree = 1857 sbinfo->max_blocks - percpu_counter_sum(&sbinfo->used_blocks); 1858 } 1859 if (sbinfo->max_inodes) { 1860 buf->f_files = sbinfo->max_inodes; 1861 buf->f_ffree = sbinfo->free_inodes; 1862 } 1863 /* else leave those fields 0 like simple_statfs */ 1864 return 0; 1865} 1866 1867/* 1868 * File creation. Allocate an inode, and we're done.. 1869 */ 1870static int 1871shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev) 1872{ 1873 struct inode *inode; 1874 int error = -ENOSPC; 1875 1876 inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE); 1877 if (inode) { 1878 error = security_inode_init_security(inode, dir, 1879 &dentry->d_name, NULL, 1880 NULL, NULL); 1881 if (error) { 1882 if (error != -EOPNOTSUPP) { 1883 iput(inode); 1884 return error; 1885 } 1886 } 1887#ifdef CONFIG_TMPFS_POSIX_ACL 1888 error = generic_acl_init(inode, dir); 1889 if (error) { 1890 iput(inode); 1891 return error; 1892 } 1893#else 1894 error = 0; 1895#endif 1896 dir->i_size += BOGO_DIRENT_SIZE; 1897 dir->i_ctime = dir->i_mtime = CURRENT_TIME; 1898 d_instantiate(dentry, inode); 1899 dget(dentry); /* Extra count - pin the dentry in core */ 1900 } 1901 return error; 1902} 1903 1904static int shmem_mkdir(struct inode *dir, struct dentry *dentry, int mode) 1905{ 1906 int error; 1907 1908 if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0))) 1909 return error; 1910 inc_nlink(dir); 1911 return 0; 1912} 1913 1914static int shmem_create(struct inode *dir, struct dentry *dentry, int mode, 1915 struct nameidata *nd) 1916{ 1917 return shmem_mknod(dir, dentry, mode | S_IFREG, 0); 1918} 1919 1920/* 1921 * Link a file.. 1922 */ 1923static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) 1924{ 1925 struct inode *inode = old_dentry->d_inode; 1926 int ret; 1927 1928 /* 1929 * No ordinary (disk based) filesystem counts links as inodes; 1930 * but each new link needs a new dentry, pinning lowmem, and 1931 * tmpfs dentries cannot be pruned until they are unlinked. 1932 */ 1933 ret = shmem_reserve_inode(inode->i_sb); 1934 if (ret) 1935 goto out; 1936 1937 dir->i_size += BOGO_DIRENT_SIZE; 1938 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME; 1939 inc_nlink(inode); 1940 ihold(inode); /* New dentry reference */ 1941 dget(dentry); /* Extra pinning count for the created dentry */ 1942 d_instantiate(dentry, inode); 1943out: 1944 return ret; 1945} 1946 1947static int shmem_unlink(struct inode *dir, struct dentry *dentry) 1948{ 1949 struct inode *inode = dentry->d_inode; 1950 1951 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode)) 1952 shmem_free_inode(inode->i_sb); 1953 1954 dir->i_size -= BOGO_DIRENT_SIZE; 1955 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME; 1956 drop_nlink(inode); 1957 dput(dentry); /* Undo the count from "create" - this does all the work */ 1958 return 0; 1959} 1960 1961static int shmem_rmdir(struct inode *dir, struct dentry *dentry) 1962{ 1963 if (!simple_empty(dentry)) 1964 return -ENOTEMPTY; 1965 1966 drop_nlink(dentry->d_inode); 1967 drop_nlink(dir); 1968 return shmem_unlink(dir, dentry); 1969} 1970 1971/* 1972 * The VFS layer already does all the dentry stuff for rename, 1973 * we just have to decrement the usage count for the target if 1974 * it exists so that the VFS layer correctly free's it when it 1975 * gets overwritten. 1976 */ 1977static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry) 1978{ 1979 struct inode *inode = old_dentry->d_inode; 1980 int they_are_dirs = S_ISDIR(inode->i_mode); 1981 1982 if (!simple_empty(new_dentry)) 1983 return -ENOTEMPTY; 1984 1985 if (new_dentry->d_inode) { 1986 (void) shmem_unlink(new_dir, new_dentry); 1987 if (they_are_dirs) 1988 drop_nlink(old_dir); 1989 } else if (they_are_dirs) { 1990 drop_nlink(old_dir); 1991 inc_nlink(new_dir); 1992 } 1993 1994 old_dir->i_size -= BOGO_DIRENT_SIZE; 1995 new_dir->i_size += BOGO_DIRENT_SIZE; 1996 old_dir->i_ctime = old_dir->i_mtime = 1997 new_dir->i_ctime = new_dir->i_mtime = 1998 inode->i_ctime = CURRENT_TIME; 1999 return 0; 2000} 2001 2002static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname) 2003{ 2004 int error; 2005 int len; 2006 struct inode *inode; 2007 struct page *page = NULL; 2008 char *kaddr; 2009 struct shmem_inode_info *info; 2010 2011 len = strlen(symname) + 1; 2012 if (len > PAGE_CACHE_SIZE) 2013 return -ENAMETOOLONG; 2014 2015 inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE); 2016 if (!inode) 2017 return -ENOSPC; 2018 2019 error = security_inode_init_security(inode, dir, &dentry->d_name, NULL, 2020 NULL, NULL); 2021 if (error) { 2022 if (error != -EOPNOTSUPP) { 2023 iput(inode); 2024 return error; 2025 } 2026 error = 0; 2027 } 2028 2029 info = SHMEM_I(inode); 2030 inode->i_size = len-1; 2031 if (len <= SHMEM_SYMLINK_INLINE_LEN) { 2032 /* do it inline */ 2033 memcpy(info->inline_symlink, symname, len); 2034 inode->i_op = &shmem_symlink_inline_operations; 2035 } else { 2036 error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL); 2037 if (error) { 2038 iput(inode); 2039 return error; 2040 } 2041 inode->i_mapping->a_ops = &shmem_aops; 2042 inode->i_op = &shmem_symlink_inode_operations; 2043 kaddr = kmap_atomic(page, KM_USER0); 2044 memcpy(kaddr, symname, len); 2045 kunmap_atomic(kaddr, KM_USER0); 2046 set_page_dirty(page); 2047 unlock_page(page); 2048 page_cache_release(page); 2049 } 2050 dir->i_size += BOGO_DIRENT_SIZE; 2051 dir->i_ctime = dir->i_mtime = CURRENT_TIME; 2052 d_instantiate(dentry, inode); 2053 dget(dentry); 2054 return 0; 2055} 2056 2057static void *shmem_follow_link_inline(struct dentry *dentry, struct nameidata *nd) 2058{ 2059 nd_set_link(nd, SHMEM_I(dentry->d_inode)->inline_symlink); 2060 return NULL; 2061} 2062 2063static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd) 2064{ 2065 struct page *page = NULL; 2066 int res = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL); 2067 nd_set_link(nd, res ? ERR_PTR(res) : kmap(page)); 2068 if (page) 2069 unlock_page(page); 2070 return page; 2071} 2072 2073static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie) 2074{ 2075 if (!IS_ERR(nd_get_link(nd))) { 2076 struct page *page = cookie; 2077 kunmap(page); 2078 mark_page_accessed(page); 2079 page_cache_release(page); 2080 } 2081} 2082 2083#ifdef CONFIG_TMPFS_XATTR 2084/* 2085 * Superblocks without xattr inode operations may get some security.* xattr 2086 * support from the LSM "for free". As soon as we have any other xattrs 2087 * like ACLs, we also need to implement the security.* handlers at 2088 * filesystem level, though. 2089 */ 2090 2091static int shmem_xattr_get(struct dentry *dentry, const char *name, 2092 void *buffer, size_t size) 2093{ 2094 struct shmem_inode_info *info; 2095 struct shmem_xattr *xattr; 2096 int ret = -ENODATA; 2097 2098 info = SHMEM_I(dentry->d_inode); 2099 2100 spin_lock(&info->lock); 2101 list_for_each_entry(xattr, &info->xattr_list, list) { 2102 if (strcmp(name, xattr->name)) 2103 continue; 2104 2105 ret = xattr->size; 2106 if (buffer) { 2107 if (size < xattr->size) 2108 ret = -ERANGE; 2109 else 2110 memcpy(buffer, xattr->value, xattr->size); 2111 } 2112 break; 2113 } 2114 spin_unlock(&info->lock); 2115 return ret; 2116} 2117 2118static int shmem_xattr_set(struct dentry *dentry, const char *name, 2119 const void *value, size_t size, int flags) 2120{ 2121 struct inode *inode = dentry->d_inode; 2122 struct shmem_inode_info *info = SHMEM_I(inode); 2123 struct shmem_xattr *xattr; 2124 struct shmem_xattr *new_xattr = NULL; 2125 size_t len; 2126 int err = 0; 2127 2128 /* value == NULL means remove */ 2129 if (value) { 2130 /* wrap around? */ 2131 len = sizeof(*new_xattr) + size; 2132 if (len <= sizeof(*new_xattr)) 2133 return -ENOMEM; 2134 2135 new_xattr = kmalloc(len, GFP_KERNEL); 2136 if (!new_xattr) 2137 return -ENOMEM; 2138 2139 new_xattr->name = kstrdup(name, GFP_KERNEL); 2140 if (!new_xattr->name) { 2141 kfree(new_xattr); 2142 return -ENOMEM; 2143 } 2144 2145 new_xattr->size = size; 2146 memcpy(new_xattr->value, value, size); 2147 } 2148 2149 spin_lock(&info->lock); 2150 list_for_each_entry(xattr, &info->xattr_list, list) { 2151 if (!strcmp(name, xattr->name)) { 2152 if (flags & XATTR_CREATE) { 2153 xattr = new_xattr; 2154 err = -EEXIST; 2155 } else if (new_xattr) { 2156 list_replace(&xattr->list, &new_xattr->list); 2157 } else { 2158 list_del(&xattr->list); 2159 } 2160 goto out; 2161 } 2162 } 2163 if (flags & XATTR_REPLACE) { 2164 xattr = new_xattr; 2165 err = -ENODATA; 2166 } else { 2167 list_add(&new_xattr->list, &info->xattr_list); 2168 xattr = NULL; 2169 } 2170out: 2171 spin_unlock(&info->lock); 2172 if (xattr) 2173 kfree(xattr->name); 2174 kfree(xattr); 2175 return err; 2176} 2177 2178 2179static const struct xattr_handler *shmem_xattr_handlers[] = { 2180#ifdef CONFIG_TMPFS_POSIX_ACL 2181 &generic_acl_access_handler, 2182 &generic_acl_default_handler, 2183#endif 2184 NULL 2185}; 2186 2187static int shmem_xattr_validate(const char *name) 2188{ 2189 struct { const char *prefix; size_t len; } arr[] = { 2190 { XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN }, 2191 { XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN } 2192 }; 2193 int i; 2194 2195 for (i = 0; i < ARRAY_SIZE(arr); i++) { 2196 size_t preflen = arr[i].len; 2197 if (strncmp(name, arr[i].prefix, preflen) == 0) { 2198 if (!name[preflen]) 2199 return -EINVAL; 2200 return 0; 2201 } 2202 } 2203 return -EOPNOTSUPP; 2204} 2205 2206static ssize_t shmem_getxattr(struct dentry *dentry, const char *name, 2207 void *buffer, size_t size) 2208{ 2209 int err; 2210 2211 /* 2212 * If this is a request for a synthetic attribute in the system.* 2213 * namespace use the generic infrastructure to resolve a handler 2214 * for it via sb->s_xattr. 2215 */ 2216 if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN)) 2217 return generic_getxattr(dentry, name, buffer, size); 2218 2219 err = shmem_xattr_validate(name); 2220 if (err) 2221 return err; 2222 2223 return shmem_xattr_get(dentry, name, buffer, size); 2224} 2225 2226static int shmem_setxattr(struct dentry *dentry, const char *name, 2227 const void *value, size_t size, int flags) 2228{ 2229 int err; 2230 2231 /* 2232 * If this is a request for a synthetic attribute in the system.* 2233 * namespace use the generic infrastructure to resolve a handler 2234 * for it via sb->s_xattr. 2235 */ 2236 if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN)) 2237 return generic_setxattr(dentry, name, value, size, flags); 2238 2239 err = shmem_xattr_validate(name); 2240 if (err) 2241 return err; 2242 2243 if (size == 0) 2244 value = ""; /* empty EA, do not remove */ 2245 2246 return shmem_xattr_set(dentry, name, value, size, flags); 2247 2248} 2249 2250static int shmem_removexattr(struct dentry *dentry, const char *name) 2251{ 2252 int err; 2253 2254 /* 2255 * If this is a request for a synthetic attribute in the system.* 2256 * namespace use the generic infrastructure to resolve a handler 2257 * for it via sb->s_xattr. 2258 */ 2259 if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN)) 2260 return generic_removexattr(dentry, name); 2261 2262 err = shmem_xattr_validate(name); 2263 if (err) 2264 return err; 2265 2266 return shmem_xattr_set(dentry, name, NULL, 0, XATTR_REPLACE); 2267} 2268 2269static bool xattr_is_trusted(const char *name) 2270{ 2271 return !strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN); 2272} 2273 2274static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size) 2275{ 2276 bool trusted = capable(CAP_SYS_ADMIN); 2277 struct shmem_xattr *xattr; 2278 struct shmem_inode_info *info; 2279 size_t used = 0; 2280 2281 info = SHMEM_I(dentry->d_inode); 2282 2283 spin_lock(&info->lock); 2284 list_for_each_entry(xattr, &info->xattr_list, list) { 2285 size_t len; 2286 2287 /* skip "trusted." attributes for unprivileged callers */ 2288 if (!trusted && xattr_is_trusted(xattr->name)) 2289 continue; 2290 2291 len = strlen(xattr->name) + 1; 2292 used += len; 2293 if (buffer) { 2294 if (size < used) { 2295 used = -ERANGE; 2296 break; 2297 } 2298 memcpy(buffer, xattr->name, len); 2299 buffer += len; 2300 } 2301 } 2302 spin_unlock(&info->lock); 2303 2304 return used; 2305} 2306#endif /* CONFIG_TMPFS_XATTR */ 2307 2308static const struct inode_operations shmem_symlink_inline_operations = { 2309 .readlink = generic_readlink, 2310 .follow_link = shmem_follow_link_inline, 2311#ifdef CONFIG_TMPFS_XATTR 2312 .setxattr = shmem_setxattr, 2313 .getxattr = shmem_getxattr, 2314 .listxattr = shmem_listxattr, 2315 .removexattr = shmem_removexattr, 2316#endif 2317}; 2318 2319static const struct inode_operations shmem_symlink_inode_operations = { 2320 .readlink = generic_readlink, 2321 .follow_link = shmem_follow_link, 2322 .put_link = shmem_put_link, 2323#ifdef CONFIG_TMPFS_XATTR 2324 .setxattr = shmem_setxattr, 2325 .getxattr = shmem_getxattr, 2326 .listxattr = shmem_listxattr, 2327 .removexattr = shmem_removexattr, 2328#endif 2329}; 2330 2331static struct dentry *shmem_get_parent(struct dentry *child) 2332{ 2333 return ERR_PTR(-ESTALE); 2334} 2335 2336static int shmem_match(struct inode *ino, void *vfh) 2337{ 2338 __u32 *fh = vfh; 2339 __u64 inum = fh[2]; 2340 inum = (inum << 32) | fh[1]; 2341 return ino->i_ino == inum && fh[0] == ino->i_generation; 2342} 2343 2344static struct dentry *shmem_fh_to_dentry(struct super_block *sb, 2345 struct fid *fid, int fh_len, int fh_type) 2346{ 2347 struct inode *inode; 2348 struct dentry *dentry = NULL; 2349 u64 inum = fid->raw[2]; 2350 inum = (inum << 32) | fid->raw[1]; 2351 2352 if (fh_len < 3) 2353 return NULL; 2354 2355 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]), 2356 shmem_match, fid->raw); 2357 if (inode) { 2358 dentry = d_find_alias(inode); 2359 iput(inode); 2360 } 2361 2362 return dentry; 2363} 2364 2365static int shmem_encode_fh(struct dentry *dentry, __u32 *fh, int *len, 2366 int connectable) 2367{ 2368 struct inode *inode = dentry->d_inode; 2369 2370 if (*len < 3) { 2371 *len = 3; 2372 return 255; 2373 } 2374 2375 if (inode_unhashed(inode)) { 2376 /* Unfortunately insert_inode_hash is not idempotent, 2377 * so as we hash inodes here rather than at creation 2378 * time, we need a lock to ensure we only try 2379 * to do it once 2380 */ 2381 static DEFINE_SPINLOCK(lock); 2382 spin_lock(&lock); 2383 if (inode_unhashed(inode)) 2384 __insert_inode_hash(inode, 2385 inode->i_ino + inode->i_generation); 2386 spin_unlock(&lock); 2387 } 2388 2389 fh[0] = inode->i_generation; 2390 fh[1] = inode->i_ino; 2391 fh[2] = ((__u64)inode->i_ino) >> 32; 2392 2393 *len = 3; 2394 return 1; 2395} 2396 2397static const struct export_operations shmem_export_ops = { 2398 .get_parent = shmem_get_parent, 2399 .encode_fh = shmem_encode_fh, 2400 .fh_to_dentry = shmem_fh_to_dentry, 2401}; 2402 2403static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo, 2404 bool remount) 2405{ 2406 char *this_char, *value, *rest; 2407 2408 while (options != NULL) { 2409 this_char = options; 2410 for (;;) { 2411 /* 2412 * NUL-terminate this option: unfortunately, 2413 * mount options form a comma-separated list, 2414 * but mpol's nodelist may also contain commas. 2415 */ 2416 options = strchr(options, ','); 2417 if (options == NULL) 2418 break; 2419 options++; 2420 if (!isdigit(*options)) { 2421 options[-1] = '\0'; 2422 break; 2423 } 2424 } 2425 if (!*this_char) 2426 continue; 2427 if ((value = strchr(this_char,'=')) != NULL) { 2428 *value++ = 0; 2429 } else { 2430 printk(KERN_ERR 2431 "tmpfs: No value for mount option '%s'\n", 2432 this_char); 2433 return 1; 2434 } 2435 2436 if (!strcmp(this_char,"size")) { 2437 unsigned long long size; 2438 size = memparse(value,&rest); 2439 if (*rest == '%') { 2440 size <<= PAGE_SHIFT; 2441 size *= totalram_pages; 2442 do_div(size, 100); 2443 rest++; 2444 } 2445 if (*rest) 2446 goto bad_val; 2447 sbinfo->max_blocks = 2448 DIV_ROUND_UP(size, PAGE_CACHE_SIZE); 2449 } else if (!strcmp(this_char,"nr_blocks")) { 2450 sbinfo->max_blocks = memparse(value, &rest); 2451 if (*rest) 2452 goto bad_val; 2453 } else if (!strcmp(this_char,"nr_inodes")) { 2454 sbinfo->max_inodes = memparse(value, &rest); 2455 if (*rest) 2456 goto bad_val; 2457 } else if (!strcmp(this_char,"mode")) { 2458 if (remount) 2459 continue; 2460 sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777; 2461 if (*rest) 2462 goto bad_val; 2463 } else if (!strcmp(this_char,"uid")) { 2464 if (remount) 2465 continue; 2466 sbinfo->uid = simple_strtoul(value, &rest, 0); 2467 if (*rest) 2468 goto bad_val; 2469 } else if (!strcmp(this_char,"gid")) { 2470 if (remount) 2471 continue; 2472 sbinfo->gid = simple_strtoul(value, &rest, 0); 2473 if (*rest) 2474 goto bad_val; 2475 } else if (!strcmp(this_char,"mpol")) { 2476 if (mpol_parse_str(value, &sbinfo->mpol, 1)) 2477 goto bad_val; 2478 } else { 2479 printk(KERN_ERR "tmpfs: Bad mount option %s\n", 2480 this_char); 2481 return 1; 2482 } 2483 } 2484 return 0; 2485 2486bad_val: 2487 printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n", 2488 value, this_char); 2489 return 1; 2490 2491} 2492 2493static int shmem_remount_fs(struct super_block *sb, int *flags, char *data) 2494{ 2495 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 2496 struct shmem_sb_info config = *sbinfo; 2497 unsigned long inodes; 2498 int error = -EINVAL; 2499 2500 if (shmem_parse_options(data, &config, true)) 2501 return error; 2502 2503 spin_lock(&sbinfo->stat_lock); 2504 inodes = sbinfo->max_inodes - sbinfo->free_inodes; 2505 if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0) 2506 goto out; 2507 if (config.max_inodes < inodes) 2508 goto out; 2509 /* 2510 * Those tests also disallow limited->unlimited while any are in 2511 * use, so i_blocks will always be zero when max_blocks is zero; 2512 * but we must separately disallow unlimited->limited, because 2513 * in that case we have no record of how much is already in use. 2514 */ 2515 if (config.max_blocks && !sbinfo->max_blocks) 2516 goto out; 2517 if (config.max_inodes && !sbinfo->max_inodes) 2518 goto out; 2519 2520 error = 0; 2521 sbinfo->max_blocks = config.max_blocks; 2522 sbinfo->max_inodes = config.max_inodes; 2523 sbinfo->free_inodes = config.max_inodes - inodes; 2524 2525 mpol_put(sbinfo->mpol); 2526 sbinfo->mpol = config.mpol; /* transfers initial ref */ 2527out: 2528 spin_unlock(&sbinfo->stat_lock); 2529 return error; 2530} 2531 2532static int shmem_show_options(struct seq_file *seq, struct vfsmount *vfs) 2533{ 2534 struct shmem_sb_info *sbinfo = SHMEM_SB(vfs->mnt_sb); 2535 2536 if (sbinfo->max_blocks != shmem_default_max_blocks()) 2537 seq_printf(seq, ",size=%luk", 2538 sbinfo->max_blocks << (PAGE_CACHE_SHIFT - 10)); 2539 if (sbinfo->max_inodes != shmem_default_max_inodes()) 2540 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes); 2541 if (sbinfo->mode != (S_IRWXUGO | S_ISVTX)) 2542 seq_printf(seq, ",mode=%03o", sbinfo->mode); 2543 if (sbinfo->uid != 0) 2544 seq_printf(seq, ",uid=%u", sbinfo->uid); 2545 if (sbinfo->gid != 0) 2546 seq_printf(seq, ",gid=%u", sbinfo->gid); 2547 shmem_show_mpol(seq, sbinfo->mpol); 2548 return 0; 2549} 2550#endif /* CONFIG_TMPFS */ 2551 2552static void shmem_put_super(struct super_block *sb) 2553{ 2554 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 2555 2556 percpu_counter_destroy(&sbinfo->used_blocks); 2557 kfree(sbinfo); 2558 sb->s_fs_info = NULL; 2559} 2560 2561int shmem_fill_super(struct super_block *sb, void *data, int silent) 2562{ 2563 struct inode *inode; 2564 struct dentry *root; 2565 struct shmem_sb_info *sbinfo; 2566 int err = -ENOMEM; 2567 2568 /* Round up to L1_CACHE_BYTES to resist false sharing */ 2569 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info), 2570 L1_CACHE_BYTES), GFP_KERNEL); 2571 if (!sbinfo) 2572 return -ENOMEM; 2573 2574 sbinfo->mode = S_IRWXUGO | S_ISVTX; 2575 sbinfo->uid = current_fsuid(); 2576 sbinfo->gid = current_fsgid(); 2577 sb->s_fs_info = sbinfo; 2578 2579#ifdef CONFIG_TMPFS 2580 /* 2581 * Per default we only allow half of the physical ram per 2582 * tmpfs instance, limiting inodes to one per page of lowmem; 2583 * but the internal instance is left unlimited. 2584 */ 2585 if (!(sb->s_flags & MS_NOUSER)) { 2586 sbinfo->max_blocks = shmem_default_max_blocks(); 2587 sbinfo->max_inodes = shmem_default_max_inodes(); 2588 if (shmem_parse_options(data, sbinfo, false)) { 2589 err = -EINVAL; 2590 goto failed; 2591 } 2592 } 2593 sb->s_export_op = &shmem_export_ops; 2594#else 2595 sb->s_flags |= MS_NOUSER; 2596#endif 2597 2598 spin_lock_init(&sbinfo->stat_lock); 2599 if (percpu_counter_init(&sbinfo->used_blocks, 0)) 2600 goto failed; 2601 sbinfo->free_inodes = sbinfo->max_inodes; 2602 2603 sb->s_maxbytes = SHMEM_MAX_BYTES; 2604 sb->s_blocksize = PAGE_CACHE_SIZE; 2605 sb->s_blocksize_bits = PAGE_CACHE_SHIFT; 2606 sb->s_magic = TMPFS_MAGIC; 2607 sb->s_op = &shmem_ops; 2608 sb->s_time_gran = 1; 2609#ifdef CONFIG_TMPFS_XATTR 2610 sb->s_xattr = shmem_xattr_handlers; 2611#endif 2612#ifdef CONFIG_TMPFS_POSIX_ACL 2613 sb->s_flags |= MS_POSIXACL; 2614#endif 2615 2616 inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE); 2617 if (!inode) 2618 goto failed; 2619 inode->i_uid = sbinfo->uid; 2620 inode->i_gid = sbinfo->gid; 2621 root = d_alloc_root(inode); 2622 if (!root) 2623 goto failed_iput; 2624 sb->s_root = root; 2625 return 0; 2626 2627failed_iput: 2628 iput(inode); 2629failed: 2630 shmem_put_super(sb); 2631 return err; 2632} 2633 2634static struct kmem_cache *shmem_inode_cachep; 2635 2636static struct inode *shmem_alloc_inode(struct super_block *sb) 2637{ 2638 struct shmem_inode_info *p; 2639 p = (struct shmem_inode_info *)kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL); 2640 if (!p) 2641 return NULL; 2642 return &p->vfs_inode; 2643} 2644 2645static void shmem_i_callback(struct rcu_head *head) 2646{ 2647 struct inode *inode = container_of(head, struct inode, i_rcu); 2648 INIT_LIST_HEAD(&inode->i_dentry); 2649 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode)); 2650} 2651 2652static void shmem_destroy_inode(struct inode *inode) 2653{ 2654 if ((inode->i_mode & S_IFMT) == S_IFREG) { 2655 /* only struct inode is valid if it's an inline symlink */ 2656 mpol_free_shared_policy(&SHMEM_I(inode)->policy); 2657 } 2658 call_rcu(&inode->i_rcu, shmem_i_callback); 2659} 2660 2661static void init_once(void *foo) 2662{ 2663 struct shmem_inode_info *p = (struct shmem_inode_info *) foo; 2664 2665 inode_init_once(&p->vfs_inode); 2666} 2667 2668static int init_inodecache(void) 2669{ 2670 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache", 2671 sizeof(struct shmem_inode_info), 2672 0, SLAB_PANIC, init_once); 2673 return 0; 2674} 2675 2676static void destroy_inodecache(void) 2677{ 2678 kmem_cache_destroy(shmem_inode_cachep); 2679} 2680 2681static const struct address_space_operations shmem_aops = { 2682 .writepage = shmem_writepage, 2683 .set_page_dirty = __set_page_dirty_no_writeback, 2684#ifdef CONFIG_TMPFS 2685 .readpage = shmem_readpage, 2686 .write_begin = shmem_write_begin, 2687 .write_end = shmem_write_end, 2688#endif 2689 .migratepage = migrate_page, 2690 .error_remove_page = generic_error_remove_page, 2691}; 2692 2693static const struct file_operations shmem_file_operations = { 2694 .mmap = shmem_mmap, 2695#ifdef CONFIG_TMPFS 2696 .llseek = generic_file_llseek, 2697 .read = do_sync_read, 2698 .write = do_sync_write, 2699 .aio_read = shmem_file_aio_read, 2700 .aio_write = generic_file_aio_write, 2701 .fsync = noop_fsync, 2702 .splice_read = generic_file_splice_read, 2703 .splice_write = generic_file_splice_write, 2704#endif 2705}; 2706 2707static const struct inode_operations shmem_inode_operations = { 2708 .setattr = shmem_notify_change, 2709 .truncate_range = shmem_truncate_range, 2710#ifdef CONFIG_TMPFS_XATTR 2711 .setxattr = shmem_setxattr, 2712 .getxattr = shmem_getxattr, 2713 .listxattr = shmem_listxattr, 2714 .removexattr = shmem_removexattr, 2715#endif 2716#ifdef CONFIG_TMPFS_POSIX_ACL 2717 .check_acl = generic_check_acl, 2718#endif 2719 2720}; 2721 2722static const struct inode_operations shmem_dir_inode_operations = { 2723#ifdef CONFIG_TMPFS 2724 .create = shmem_create, 2725 .lookup = simple_lookup, 2726 .link = shmem_link, 2727 .unlink = shmem_unlink, 2728 .symlink = shmem_symlink, 2729 .mkdir = shmem_mkdir, 2730 .rmdir = shmem_rmdir, 2731 .mknod = shmem_mknod, 2732 .rename = shmem_rename, 2733#endif 2734#ifdef CONFIG_TMPFS_XATTR 2735 .setxattr = shmem_setxattr, 2736 .getxattr = shmem_getxattr, 2737 .listxattr = shmem_listxattr, 2738 .removexattr = shmem_removexattr, 2739#endif 2740#ifdef CONFIG_TMPFS_POSIX_ACL 2741 .setattr = shmem_notify_change, 2742 .check_acl = generic_check_acl, 2743#endif 2744}; 2745 2746static const struct inode_operations shmem_special_inode_operations = { 2747#ifdef CONFIG_TMPFS_XATTR 2748 .setxattr = shmem_setxattr, 2749 .getxattr = shmem_getxattr, 2750 .listxattr = shmem_listxattr, 2751 .removexattr = shmem_removexattr, 2752#endif 2753#ifdef CONFIG_TMPFS_POSIX_ACL 2754 .setattr = shmem_notify_change, 2755 .check_acl = generic_check_acl, 2756#endif 2757}; 2758 2759static const struct super_operations shmem_ops = { 2760 .alloc_inode = shmem_alloc_inode, 2761 .destroy_inode = shmem_destroy_inode, 2762#ifdef CONFIG_TMPFS 2763 .statfs = shmem_statfs, 2764 .remount_fs = shmem_remount_fs, 2765 .show_options = shmem_show_options, 2766#endif 2767 .evict_inode = shmem_evict_inode, 2768 .drop_inode = generic_delete_inode, 2769 .put_super = shmem_put_super, 2770}; 2771 2772static const struct vm_operations_struct shmem_vm_ops = { 2773 .fault = shmem_fault, 2774#ifdef CONFIG_NUMA 2775 .set_policy = shmem_set_policy, 2776 .get_policy = shmem_get_policy, 2777#endif 2778}; 2779 2780 2781static struct dentry *shmem_mount(struct file_system_type *fs_type, 2782 int flags, const char *dev_name, void *data) 2783{ 2784 return mount_nodev(fs_type, flags, data, shmem_fill_super); 2785} 2786 2787static struct file_system_type tmpfs_fs_type = { 2788 .owner = THIS_MODULE, 2789 .name = "tmpfs", 2790 .mount = shmem_mount, 2791 .kill_sb = kill_litter_super, 2792}; 2793 2794int __init init_tmpfs(void) 2795{ 2796 int error; 2797 2798 error = bdi_init(&shmem_backing_dev_info); 2799 if (error) 2800 goto out4; 2801 2802 error = init_inodecache(); 2803 if (error) 2804 goto out3; 2805 2806 error = register_filesystem(&tmpfs_fs_type); 2807 if (error) { 2808 printk(KERN_ERR "Could not register tmpfs\n"); 2809 goto out2; 2810 } 2811 2812 shm_mnt = vfs_kern_mount(&tmpfs_fs_type, MS_NOUSER, 2813 tmpfs_fs_type.name, NULL); 2814 if (IS_ERR(shm_mnt)) { 2815 error = PTR_ERR(shm_mnt); 2816 printk(KERN_ERR "Could not kern_mount tmpfs\n"); 2817 goto out1; 2818 } 2819 return 0; 2820 2821out1: 2822 unregister_filesystem(&tmpfs_fs_type); 2823out2: 2824 destroy_inodecache(); 2825out3: 2826 bdi_destroy(&shmem_backing_dev_info); 2827out4: 2828 shm_mnt = ERR_PTR(error); 2829 return error; 2830} 2831 2832#ifdef CONFIG_CGROUP_MEM_RES_CTLR 2833/** 2834 * mem_cgroup_get_shmem_target - find a page or entry assigned to the shmem file 2835 * @inode: the inode to be searched 2836 * @pgoff: the offset to be searched 2837 * @pagep: the pointer for the found page to be stored 2838 * @ent: the pointer for the found swap entry to be stored 2839 * 2840 * If a page is found, refcount of it is incremented. Callers should handle 2841 * these refcount. 2842 */ 2843void mem_cgroup_get_shmem_target(struct inode *inode, pgoff_t pgoff, 2844 struct page **pagep, swp_entry_t *ent) 2845{ 2846 swp_entry_t entry = { .val = 0 }, *ptr; 2847 struct page *page = NULL; 2848 struct shmem_inode_info *info = SHMEM_I(inode); 2849 2850 if ((pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode)) 2851 goto out; 2852 2853 spin_lock(&info->lock); 2854 ptr = shmem_swp_entry(info, pgoff, NULL); 2855#ifdef CONFIG_SWAP 2856 if (ptr && ptr->val) { 2857 entry.val = ptr->val; 2858 page = find_get_page(&swapper_space, entry.val); 2859 } else 2860#endif 2861 page = find_get_page(inode->i_mapping, pgoff); 2862 if (ptr) 2863 shmem_swp_unmap(ptr); 2864 spin_unlock(&info->lock); 2865out: 2866 *pagep = page; 2867 *ent = entry; 2868} 2869#endif 2870 2871#else /* !CONFIG_SHMEM */ 2872 2873/* 2874 * tiny-shmem: simple shmemfs and tmpfs using ramfs code 2875 * 2876 * This is intended for small system where the benefits of the full 2877 * shmem code (swap-backed and resource-limited) are outweighed by 2878 * their complexity. On systems without swap this code should be 2879 * effectively equivalent, but much lighter weight. 2880 */ 2881 2882#include <linux/ramfs.h> 2883 2884static struct file_system_type tmpfs_fs_type = { 2885 .name = "tmpfs", 2886 .mount = ramfs_mount, 2887 .kill_sb = kill_litter_super, 2888}; 2889 2890int __init init_tmpfs(void) 2891{ 2892 BUG_ON(register_filesystem(&tmpfs_fs_type) != 0); 2893 2894 shm_mnt = kern_mount(&tmpfs_fs_type); 2895 BUG_ON(IS_ERR(shm_mnt)); 2896 2897 return 0; 2898} 2899 2900int shmem_unuse(swp_entry_t entry, struct page *page) 2901{ 2902 return 0; 2903} 2904 2905int shmem_lock(struct file *file, int lock, struct user_struct *user) 2906{ 2907 return 0; 2908} 2909 2910#ifdef CONFIG_CGROUP_MEM_RES_CTLR 2911/** 2912 * mem_cgroup_get_shmem_target - find a page or entry assigned to the shmem file 2913 * @inode: the inode to be searched 2914 * @pgoff: the offset to be searched 2915 * @pagep: the pointer for the found page to be stored 2916 * @ent: the pointer for the found swap entry to be stored 2917 * 2918 * If a page is found, refcount of it is incremented. Callers should handle 2919 * these refcount. 2920 */ 2921void mem_cgroup_get_shmem_target(struct inode *inode, pgoff_t pgoff, 2922 struct page **pagep, swp_entry_t *ent) 2923{ 2924 struct page *page = NULL; 2925 2926 if ((pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode)) 2927 goto out; 2928 page = find_get_page(inode->i_mapping, pgoff); 2929out: 2930 *pagep = page; 2931 *ent = (swp_entry_t){ .val = 0 }; 2932} 2933#endif 2934 2935#define shmem_vm_ops generic_file_vm_ops 2936#define shmem_file_operations ramfs_file_operations 2937#define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev) 2938#define shmem_acct_size(flags, size) 0 2939#define shmem_unacct_size(flags, size) do {} while (0) 2940#define SHMEM_MAX_BYTES MAX_LFS_FILESIZE 2941 2942#endif /* CONFIG_SHMEM */ 2943 2944/* common code */ 2945 2946/** 2947 * shmem_file_setup - get an unlinked file living in tmpfs 2948 * @name: name for dentry (to be seen in /proc/<pid>/maps 2949 * @size: size to be set for the file 2950 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 2951 */ 2952struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags) 2953{ 2954 int error; 2955 struct file *file; 2956 struct inode *inode; 2957 struct path path; 2958 struct dentry *root; 2959 struct qstr this; 2960 2961 if (IS_ERR(shm_mnt)) 2962 return (void *)shm_mnt; 2963 2964 if (size < 0 || size > SHMEM_MAX_BYTES) 2965 return ERR_PTR(-EINVAL); 2966 2967 if (shmem_acct_size(flags, size)) 2968 return ERR_PTR(-ENOMEM); 2969 2970 error = -ENOMEM; 2971 this.name = name; 2972 this.len = strlen(name); 2973 this.hash = 0; /* will go */ 2974 root = shm_mnt->mnt_root; 2975 path.dentry = d_alloc(root, &this); 2976 if (!path.dentry) 2977 goto put_memory; 2978 path.mnt = mntget(shm_mnt); 2979 2980 error = -ENOSPC; 2981 inode = shmem_get_inode(root->d_sb, NULL, S_IFREG | S_IRWXUGO, 0, flags); 2982 if (!inode) 2983 goto put_dentry; 2984 2985 d_instantiate(path.dentry, inode); 2986 inode->i_size = size; 2987 inode->i_nlink = 0; /* It is unlinked */ 2988#ifndef CONFIG_MMU 2989 error = ramfs_nommu_expand_for_mapping(inode, size); 2990 if (error) 2991 goto put_dentry; 2992#endif 2993 2994 error = -ENFILE; 2995 file = alloc_file(&path, FMODE_WRITE | FMODE_READ, 2996 &shmem_file_operations); 2997 if (!file) 2998 goto put_dentry; 2999 3000 return file; 3001 3002put_dentry: 3003 path_put(&path); 3004put_memory: 3005 shmem_unacct_size(flags, size); 3006 return ERR_PTR(error); 3007} 3008EXPORT_SYMBOL_GPL(shmem_file_setup); 3009 3010/** 3011 * shmem_zero_setup - setup a shared anonymous mapping 3012 * @vma: the vma to be mmapped is prepared by do_mmap_pgoff 3013 */ 3014int shmem_zero_setup(struct vm_area_struct *vma) 3015{ 3016 struct file *file; 3017 loff_t size = vma->vm_end - vma->vm_start; 3018 3019 file = shmem_file_setup("dev/zero", size, vma->vm_flags); 3020 if (IS_ERR(file)) 3021 return PTR_ERR(file); 3022 3023 if (vma->vm_file) 3024 fput(vma->vm_file); 3025 vma->vm_file = file; 3026 vma->vm_ops = &shmem_vm_ops; 3027 vma->vm_flags |= VM_CAN_NONLINEAR; 3028 return 0; 3029} 3030