shmem.c revision d515afe88a32e567c550e3db914f3e378f86453a
1/* 2 * Resizable virtual memory filesystem for Linux. 3 * 4 * Copyright (C) 2000 Linus Torvalds. 5 * 2000 Transmeta Corp. 6 * 2000-2001 Christoph Rohland 7 * 2000-2001 SAP AG 8 * 2002 Red Hat Inc. 9 * Copyright (C) 2002-2005 Hugh Dickins. 10 * Copyright (C) 2002-2005 VERITAS Software Corporation. 11 * Copyright (C) 2004 Andi Kleen, SuSE Labs 12 * 13 * Extended attribute support for tmpfs: 14 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net> 15 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com> 16 * 17 * tiny-shmem: 18 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com> 19 * 20 * This file is released under the GPL. 21 */ 22 23#include <linux/fs.h> 24#include <linux/init.h> 25#include <linux/vfs.h> 26#include <linux/mount.h> 27#include <linux/pagemap.h> 28#include <linux/file.h> 29#include <linux/mm.h> 30#include <linux/module.h> 31#include <linux/percpu_counter.h> 32#include <linux/swap.h> 33 34static struct vfsmount *shm_mnt; 35 36#ifdef CONFIG_SHMEM 37/* 38 * This virtual memory filesystem is heavily based on the ramfs. It 39 * extends ramfs by the ability to use swap and honor resource limits 40 * which makes it a completely usable filesystem. 41 */ 42 43#include <linux/xattr.h> 44#include <linux/exportfs.h> 45#include <linux/posix_acl.h> 46#include <linux/generic_acl.h> 47#include <linux/mman.h> 48#include <linux/string.h> 49#include <linux/slab.h> 50#include <linux/backing-dev.h> 51#include <linux/shmem_fs.h> 52#include <linux/writeback.h> 53#include <linux/blkdev.h> 54#include <linux/security.h> 55#include <linux/swapops.h> 56#include <linux/mempolicy.h> 57#include <linux/namei.h> 58#include <linux/ctype.h> 59#include <linux/migrate.h> 60#include <linux/highmem.h> 61#include <linux/seq_file.h> 62#include <linux/magic.h> 63 64#include <asm/uaccess.h> 65#include <asm/div64.h> 66#include <asm/pgtable.h> 67 68/* 69 * The maximum size of a shmem/tmpfs file is limited by the maximum size of 70 * its triple-indirect swap vector - see illustration at shmem_swp_entry(). 71 * 72 * With 4kB page size, maximum file size is just over 2TB on a 32-bit kernel, 73 * but one eighth of that on a 64-bit kernel. With 8kB page size, maximum 74 * file size is just over 4TB on a 64-bit kernel, but 16TB on a 32-bit kernel, 75 * MAX_LFS_FILESIZE being then more restrictive than swap vector layout. 76 * 77 * We use / and * instead of shifts in the definitions below, so that the swap 78 * vector can be tested with small even values (e.g. 20) for ENTRIES_PER_PAGE. 79 */ 80#define ENTRIES_PER_PAGE (PAGE_CACHE_SIZE/sizeof(unsigned long)) 81#define ENTRIES_PER_PAGEPAGE ((unsigned long long)ENTRIES_PER_PAGE*ENTRIES_PER_PAGE) 82 83#define SHMSWP_MAX_INDEX (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1)) 84#define SHMSWP_MAX_BYTES (SHMSWP_MAX_INDEX << PAGE_CACHE_SHIFT) 85 86#define SHMEM_MAX_BYTES min_t(unsigned long long, SHMSWP_MAX_BYTES, MAX_LFS_FILESIZE) 87#define SHMEM_MAX_INDEX ((unsigned long)((SHMEM_MAX_BYTES+1) >> PAGE_CACHE_SHIFT)) 88 89#define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512) 90#define VM_ACCT(size) (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT) 91 92/* info->flags needs VM_flags to handle pagein/truncate races efficiently */ 93#define SHMEM_PAGEIN VM_READ 94#define SHMEM_TRUNCATE VM_WRITE 95 96/* Definition to limit shmem_truncate's steps between cond_rescheds */ 97#define LATENCY_LIMIT 64 98 99/* Pretend that each entry is of this size in directory's i_size */ 100#define BOGO_DIRENT_SIZE 20 101 102struct shmem_xattr { 103 struct list_head list; /* anchored by shmem_inode_info->xattr_list */ 104 char *name; /* xattr name */ 105 size_t size; 106 char value[0]; 107}; 108 109/* Flag allocation requirements to shmem_getpage and shmem_swp_alloc */ 110enum sgp_type { 111 SGP_READ, /* don't exceed i_size, don't allocate page */ 112 SGP_CACHE, /* don't exceed i_size, may allocate page */ 113 SGP_DIRTY, /* like SGP_CACHE, but set new page dirty */ 114 SGP_WRITE, /* may exceed i_size, may allocate page */ 115}; 116 117#ifdef CONFIG_TMPFS 118static unsigned long shmem_default_max_blocks(void) 119{ 120 return totalram_pages / 2; 121} 122 123static unsigned long shmem_default_max_inodes(void) 124{ 125 return min(totalram_pages - totalhigh_pages, totalram_pages / 2); 126} 127#endif 128 129static int shmem_getpage(struct inode *inode, unsigned long idx, 130 struct page **pagep, enum sgp_type sgp, int *type); 131 132static inline struct page *shmem_dir_alloc(gfp_t gfp_mask) 133{ 134 /* 135 * The above definition of ENTRIES_PER_PAGE, and the use of 136 * BLOCKS_PER_PAGE on indirect pages, assume PAGE_CACHE_SIZE: 137 * might be reconsidered if it ever diverges from PAGE_SIZE. 138 * 139 * Mobility flags are masked out as swap vectors cannot move 140 */ 141 return alloc_pages((gfp_mask & ~GFP_MOVABLE_MASK) | __GFP_ZERO, 142 PAGE_CACHE_SHIFT-PAGE_SHIFT); 143} 144 145static inline void shmem_dir_free(struct page *page) 146{ 147 __free_pages(page, PAGE_CACHE_SHIFT-PAGE_SHIFT); 148} 149 150static struct page **shmem_dir_map(struct page *page) 151{ 152 return (struct page **)kmap_atomic(page, KM_USER0); 153} 154 155static inline void shmem_dir_unmap(struct page **dir) 156{ 157 kunmap_atomic(dir, KM_USER0); 158} 159 160static swp_entry_t *shmem_swp_map(struct page *page) 161{ 162 return (swp_entry_t *)kmap_atomic(page, KM_USER1); 163} 164 165static inline void shmem_swp_balance_unmap(void) 166{ 167 /* 168 * When passing a pointer to an i_direct entry, to code which 169 * also handles indirect entries and so will shmem_swp_unmap, 170 * we must arrange for the preempt count to remain in balance. 171 * What kmap_atomic of a lowmem page does depends on config 172 * and architecture, so pretend to kmap_atomic some lowmem page. 173 */ 174 (void) kmap_atomic(ZERO_PAGE(0), KM_USER1); 175} 176 177static inline void shmem_swp_unmap(swp_entry_t *entry) 178{ 179 kunmap_atomic(entry, KM_USER1); 180} 181 182static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb) 183{ 184 return sb->s_fs_info; 185} 186 187/* 188 * shmem_file_setup pre-accounts the whole fixed size of a VM object, 189 * for shared memory and for shared anonymous (/dev/zero) mappings 190 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1), 191 * consistent with the pre-accounting of private mappings ... 192 */ 193static inline int shmem_acct_size(unsigned long flags, loff_t size) 194{ 195 return (flags & VM_NORESERVE) ? 196 0 : security_vm_enough_memory_kern(VM_ACCT(size)); 197} 198 199static inline void shmem_unacct_size(unsigned long flags, loff_t size) 200{ 201 if (!(flags & VM_NORESERVE)) 202 vm_unacct_memory(VM_ACCT(size)); 203} 204 205/* 206 * ... whereas tmpfs objects are accounted incrementally as 207 * pages are allocated, in order to allow huge sparse files. 208 * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM, 209 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM. 210 */ 211static inline int shmem_acct_block(unsigned long flags) 212{ 213 return (flags & VM_NORESERVE) ? 214 security_vm_enough_memory_kern(VM_ACCT(PAGE_CACHE_SIZE)) : 0; 215} 216 217static inline void shmem_unacct_blocks(unsigned long flags, long pages) 218{ 219 if (flags & VM_NORESERVE) 220 vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE)); 221} 222 223static const struct super_operations shmem_ops; 224static const struct address_space_operations shmem_aops; 225static const struct file_operations shmem_file_operations; 226static const struct inode_operations shmem_inode_operations; 227static const struct inode_operations shmem_dir_inode_operations; 228static const struct inode_operations shmem_special_inode_operations; 229static const struct vm_operations_struct shmem_vm_ops; 230 231static struct backing_dev_info shmem_backing_dev_info __read_mostly = { 232 .ra_pages = 0, /* No readahead */ 233 .capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED, 234}; 235 236static LIST_HEAD(shmem_swaplist); 237static DEFINE_MUTEX(shmem_swaplist_mutex); 238 239static void shmem_free_blocks(struct inode *inode, long pages) 240{ 241 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 242 if (sbinfo->max_blocks) { 243 percpu_counter_add(&sbinfo->used_blocks, -pages); 244 inode->i_blocks -= pages*BLOCKS_PER_PAGE; 245 } 246} 247 248static int shmem_reserve_inode(struct super_block *sb) 249{ 250 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 251 if (sbinfo->max_inodes) { 252 spin_lock(&sbinfo->stat_lock); 253 if (!sbinfo->free_inodes) { 254 spin_unlock(&sbinfo->stat_lock); 255 return -ENOSPC; 256 } 257 sbinfo->free_inodes--; 258 spin_unlock(&sbinfo->stat_lock); 259 } 260 return 0; 261} 262 263static void shmem_free_inode(struct super_block *sb) 264{ 265 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 266 if (sbinfo->max_inodes) { 267 spin_lock(&sbinfo->stat_lock); 268 sbinfo->free_inodes++; 269 spin_unlock(&sbinfo->stat_lock); 270 } 271} 272 273/** 274 * shmem_recalc_inode - recalculate the size of an inode 275 * @inode: inode to recalc 276 * 277 * We have to calculate the free blocks since the mm can drop 278 * undirtied hole pages behind our back. 279 * 280 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped 281 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped) 282 * 283 * It has to be called with the spinlock held. 284 */ 285static void shmem_recalc_inode(struct inode *inode) 286{ 287 struct shmem_inode_info *info = SHMEM_I(inode); 288 long freed; 289 290 freed = info->alloced - info->swapped - inode->i_mapping->nrpages; 291 if (freed > 0) { 292 info->alloced -= freed; 293 shmem_unacct_blocks(info->flags, freed); 294 shmem_free_blocks(inode, freed); 295 } 296} 297 298/** 299 * shmem_swp_entry - find the swap vector position in the info structure 300 * @info: info structure for the inode 301 * @index: index of the page to find 302 * @page: optional page to add to the structure. Has to be preset to 303 * all zeros 304 * 305 * If there is no space allocated yet it will return NULL when 306 * page is NULL, else it will use the page for the needed block, 307 * setting it to NULL on return to indicate that it has been used. 308 * 309 * The swap vector is organized the following way: 310 * 311 * There are SHMEM_NR_DIRECT entries directly stored in the 312 * shmem_inode_info structure. So small files do not need an addional 313 * allocation. 314 * 315 * For pages with index > SHMEM_NR_DIRECT there is the pointer 316 * i_indirect which points to a page which holds in the first half 317 * doubly indirect blocks, in the second half triple indirect blocks: 318 * 319 * For an artificial ENTRIES_PER_PAGE = 4 this would lead to the 320 * following layout (for SHMEM_NR_DIRECT == 16): 321 * 322 * i_indirect -> dir --> 16-19 323 * | +-> 20-23 324 * | 325 * +-->dir2 --> 24-27 326 * | +-> 28-31 327 * | +-> 32-35 328 * | +-> 36-39 329 * | 330 * +-->dir3 --> 40-43 331 * +-> 44-47 332 * +-> 48-51 333 * +-> 52-55 334 */ 335static swp_entry_t *shmem_swp_entry(struct shmem_inode_info *info, unsigned long index, struct page **page) 336{ 337 unsigned long offset; 338 struct page **dir; 339 struct page *subdir; 340 341 if (index < SHMEM_NR_DIRECT) { 342 shmem_swp_balance_unmap(); 343 return info->i_direct+index; 344 } 345 if (!info->i_indirect) { 346 if (page) { 347 info->i_indirect = *page; 348 *page = NULL; 349 } 350 return NULL; /* need another page */ 351 } 352 353 index -= SHMEM_NR_DIRECT; 354 offset = index % ENTRIES_PER_PAGE; 355 index /= ENTRIES_PER_PAGE; 356 dir = shmem_dir_map(info->i_indirect); 357 358 if (index >= ENTRIES_PER_PAGE/2) { 359 index -= ENTRIES_PER_PAGE/2; 360 dir += ENTRIES_PER_PAGE/2 + index/ENTRIES_PER_PAGE; 361 index %= ENTRIES_PER_PAGE; 362 subdir = *dir; 363 if (!subdir) { 364 if (page) { 365 *dir = *page; 366 *page = NULL; 367 } 368 shmem_dir_unmap(dir); 369 return NULL; /* need another page */ 370 } 371 shmem_dir_unmap(dir); 372 dir = shmem_dir_map(subdir); 373 } 374 375 dir += index; 376 subdir = *dir; 377 if (!subdir) { 378 if (!page || !(subdir = *page)) { 379 shmem_dir_unmap(dir); 380 return NULL; /* need a page */ 381 } 382 *dir = subdir; 383 *page = NULL; 384 } 385 shmem_dir_unmap(dir); 386 return shmem_swp_map(subdir) + offset; 387} 388 389static void shmem_swp_set(struct shmem_inode_info *info, swp_entry_t *entry, unsigned long value) 390{ 391 long incdec = value? 1: -1; 392 393 entry->val = value; 394 info->swapped += incdec; 395 if ((unsigned long)(entry - info->i_direct) >= SHMEM_NR_DIRECT) { 396 struct page *page = kmap_atomic_to_page(entry); 397 set_page_private(page, page_private(page) + incdec); 398 } 399} 400 401/** 402 * shmem_swp_alloc - get the position of the swap entry for the page. 403 * @info: info structure for the inode 404 * @index: index of the page to find 405 * @sgp: check and recheck i_size? skip allocation? 406 * 407 * If the entry does not exist, allocate it. 408 */ 409static swp_entry_t *shmem_swp_alloc(struct shmem_inode_info *info, unsigned long index, enum sgp_type sgp) 410{ 411 struct inode *inode = &info->vfs_inode; 412 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 413 struct page *page = NULL; 414 swp_entry_t *entry; 415 416 if (sgp != SGP_WRITE && 417 ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) 418 return ERR_PTR(-EINVAL); 419 420 while (!(entry = shmem_swp_entry(info, index, &page))) { 421 if (sgp == SGP_READ) 422 return shmem_swp_map(ZERO_PAGE(0)); 423 /* 424 * Test used_blocks against 1 less max_blocks, since we have 1 data 425 * page (and perhaps indirect index pages) yet to allocate: 426 * a waste to allocate index if we cannot allocate data. 427 */ 428 if (sbinfo->max_blocks) { 429 if (percpu_counter_compare(&sbinfo->used_blocks, 430 sbinfo->max_blocks - 1) >= 0) 431 return ERR_PTR(-ENOSPC); 432 percpu_counter_inc(&sbinfo->used_blocks); 433 inode->i_blocks += BLOCKS_PER_PAGE; 434 } 435 436 spin_unlock(&info->lock); 437 page = shmem_dir_alloc(mapping_gfp_mask(inode->i_mapping)); 438 spin_lock(&info->lock); 439 440 if (!page) { 441 shmem_free_blocks(inode, 1); 442 return ERR_PTR(-ENOMEM); 443 } 444 if (sgp != SGP_WRITE && 445 ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) { 446 entry = ERR_PTR(-EINVAL); 447 break; 448 } 449 if (info->next_index <= index) 450 info->next_index = index + 1; 451 } 452 if (page) { 453 /* another task gave its page, or truncated the file */ 454 shmem_free_blocks(inode, 1); 455 shmem_dir_free(page); 456 } 457 if (info->next_index <= index && !IS_ERR(entry)) 458 info->next_index = index + 1; 459 return entry; 460} 461 462/** 463 * shmem_free_swp - free some swap entries in a directory 464 * @dir: pointer to the directory 465 * @edir: pointer after last entry of the directory 466 * @punch_lock: pointer to spinlock when needed for the holepunch case 467 */ 468static int shmem_free_swp(swp_entry_t *dir, swp_entry_t *edir, 469 spinlock_t *punch_lock) 470{ 471 spinlock_t *punch_unlock = NULL; 472 swp_entry_t *ptr; 473 int freed = 0; 474 475 for (ptr = dir; ptr < edir; ptr++) { 476 if (ptr->val) { 477 if (unlikely(punch_lock)) { 478 punch_unlock = punch_lock; 479 punch_lock = NULL; 480 spin_lock(punch_unlock); 481 if (!ptr->val) 482 continue; 483 } 484 free_swap_and_cache(*ptr); 485 *ptr = (swp_entry_t){0}; 486 freed++; 487 } 488 } 489 if (punch_unlock) 490 spin_unlock(punch_unlock); 491 return freed; 492} 493 494static int shmem_map_and_free_swp(struct page *subdir, int offset, 495 int limit, struct page ***dir, spinlock_t *punch_lock) 496{ 497 swp_entry_t *ptr; 498 int freed = 0; 499 500 ptr = shmem_swp_map(subdir); 501 for (; offset < limit; offset += LATENCY_LIMIT) { 502 int size = limit - offset; 503 if (size > LATENCY_LIMIT) 504 size = LATENCY_LIMIT; 505 freed += shmem_free_swp(ptr+offset, ptr+offset+size, 506 punch_lock); 507 if (need_resched()) { 508 shmem_swp_unmap(ptr); 509 if (*dir) { 510 shmem_dir_unmap(*dir); 511 *dir = NULL; 512 } 513 cond_resched(); 514 ptr = shmem_swp_map(subdir); 515 } 516 } 517 shmem_swp_unmap(ptr); 518 return freed; 519} 520 521static void shmem_free_pages(struct list_head *next) 522{ 523 struct page *page; 524 int freed = 0; 525 526 do { 527 page = container_of(next, struct page, lru); 528 next = next->next; 529 shmem_dir_free(page); 530 freed++; 531 if (freed >= LATENCY_LIMIT) { 532 cond_resched(); 533 freed = 0; 534 } 535 } while (next); 536} 537 538void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end) 539{ 540 struct shmem_inode_info *info = SHMEM_I(inode); 541 unsigned long idx; 542 unsigned long size; 543 unsigned long limit; 544 unsigned long stage; 545 unsigned long diroff; 546 struct page **dir; 547 struct page *topdir; 548 struct page *middir; 549 struct page *subdir; 550 swp_entry_t *ptr; 551 LIST_HEAD(pages_to_free); 552 long nr_pages_to_free = 0; 553 long nr_swaps_freed = 0; 554 int offset; 555 int freed; 556 int punch_hole; 557 spinlock_t *needs_lock; 558 spinlock_t *punch_lock; 559 unsigned long upper_limit; 560 561 truncate_inode_pages_range(inode->i_mapping, start, end); 562 563 inode->i_ctime = inode->i_mtime = CURRENT_TIME; 564 idx = (start + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 565 if (idx >= info->next_index) 566 return; 567 568 spin_lock(&info->lock); 569 info->flags |= SHMEM_TRUNCATE; 570 if (likely(end == (loff_t) -1)) { 571 limit = info->next_index; 572 upper_limit = SHMEM_MAX_INDEX; 573 info->next_index = idx; 574 needs_lock = NULL; 575 punch_hole = 0; 576 } else { 577 if (end + 1 >= inode->i_size) { /* we may free a little more */ 578 limit = (inode->i_size + PAGE_CACHE_SIZE - 1) >> 579 PAGE_CACHE_SHIFT; 580 upper_limit = SHMEM_MAX_INDEX; 581 } else { 582 limit = (end + 1) >> PAGE_CACHE_SHIFT; 583 upper_limit = limit; 584 } 585 needs_lock = &info->lock; 586 punch_hole = 1; 587 } 588 589 topdir = info->i_indirect; 590 if (topdir && idx <= SHMEM_NR_DIRECT && !punch_hole) { 591 info->i_indirect = NULL; 592 nr_pages_to_free++; 593 list_add(&topdir->lru, &pages_to_free); 594 } 595 spin_unlock(&info->lock); 596 597 if (info->swapped && idx < SHMEM_NR_DIRECT) { 598 ptr = info->i_direct; 599 size = limit; 600 if (size > SHMEM_NR_DIRECT) 601 size = SHMEM_NR_DIRECT; 602 nr_swaps_freed = shmem_free_swp(ptr+idx, ptr+size, needs_lock); 603 } 604 605 /* 606 * If there are no indirect blocks or we are punching a hole 607 * below indirect blocks, nothing to be done. 608 */ 609 if (!topdir || limit <= SHMEM_NR_DIRECT) 610 goto done2; 611 612 /* 613 * The truncation case has already dropped info->lock, and we're safe 614 * because i_size and next_index have already been lowered, preventing 615 * access beyond. But in the punch_hole case, we still need to take 616 * the lock when updating the swap directory, because there might be 617 * racing accesses by shmem_getpage(SGP_CACHE), shmem_unuse_inode or 618 * shmem_writepage. However, whenever we find we can remove a whole 619 * directory page (not at the misaligned start or end of the range), 620 * we first NULLify its pointer in the level above, and then have no 621 * need to take the lock when updating its contents: needs_lock and 622 * punch_lock (either pointing to info->lock or NULL) manage this. 623 */ 624 625 upper_limit -= SHMEM_NR_DIRECT; 626 limit -= SHMEM_NR_DIRECT; 627 idx = (idx > SHMEM_NR_DIRECT)? (idx - SHMEM_NR_DIRECT): 0; 628 offset = idx % ENTRIES_PER_PAGE; 629 idx -= offset; 630 631 dir = shmem_dir_map(topdir); 632 stage = ENTRIES_PER_PAGEPAGE/2; 633 if (idx < ENTRIES_PER_PAGEPAGE/2) { 634 middir = topdir; 635 diroff = idx/ENTRIES_PER_PAGE; 636 } else { 637 dir += ENTRIES_PER_PAGE/2; 638 dir += (idx - ENTRIES_PER_PAGEPAGE/2)/ENTRIES_PER_PAGEPAGE; 639 while (stage <= idx) 640 stage += ENTRIES_PER_PAGEPAGE; 641 middir = *dir; 642 if (*dir) { 643 diroff = ((idx - ENTRIES_PER_PAGEPAGE/2) % 644 ENTRIES_PER_PAGEPAGE) / ENTRIES_PER_PAGE; 645 if (!diroff && !offset && upper_limit >= stage) { 646 if (needs_lock) { 647 spin_lock(needs_lock); 648 *dir = NULL; 649 spin_unlock(needs_lock); 650 needs_lock = NULL; 651 } else 652 *dir = NULL; 653 nr_pages_to_free++; 654 list_add(&middir->lru, &pages_to_free); 655 } 656 shmem_dir_unmap(dir); 657 dir = shmem_dir_map(middir); 658 } else { 659 diroff = 0; 660 offset = 0; 661 idx = stage; 662 } 663 } 664 665 for (; idx < limit; idx += ENTRIES_PER_PAGE, diroff++) { 666 if (unlikely(idx == stage)) { 667 shmem_dir_unmap(dir); 668 dir = shmem_dir_map(topdir) + 669 ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE; 670 while (!*dir) { 671 dir++; 672 idx += ENTRIES_PER_PAGEPAGE; 673 if (idx >= limit) 674 goto done1; 675 } 676 stage = idx + ENTRIES_PER_PAGEPAGE; 677 middir = *dir; 678 if (punch_hole) 679 needs_lock = &info->lock; 680 if (upper_limit >= stage) { 681 if (needs_lock) { 682 spin_lock(needs_lock); 683 *dir = NULL; 684 spin_unlock(needs_lock); 685 needs_lock = NULL; 686 } else 687 *dir = NULL; 688 nr_pages_to_free++; 689 list_add(&middir->lru, &pages_to_free); 690 } 691 shmem_dir_unmap(dir); 692 cond_resched(); 693 dir = shmem_dir_map(middir); 694 diroff = 0; 695 } 696 punch_lock = needs_lock; 697 subdir = dir[diroff]; 698 if (subdir && !offset && upper_limit-idx >= ENTRIES_PER_PAGE) { 699 if (needs_lock) { 700 spin_lock(needs_lock); 701 dir[diroff] = NULL; 702 spin_unlock(needs_lock); 703 punch_lock = NULL; 704 } else 705 dir[diroff] = NULL; 706 nr_pages_to_free++; 707 list_add(&subdir->lru, &pages_to_free); 708 } 709 if (subdir && page_private(subdir) /* has swap entries */) { 710 size = limit - idx; 711 if (size > ENTRIES_PER_PAGE) 712 size = ENTRIES_PER_PAGE; 713 freed = shmem_map_and_free_swp(subdir, 714 offset, size, &dir, punch_lock); 715 if (!dir) 716 dir = shmem_dir_map(middir); 717 nr_swaps_freed += freed; 718 if (offset || punch_lock) { 719 spin_lock(&info->lock); 720 set_page_private(subdir, 721 page_private(subdir) - freed); 722 spin_unlock(&info->lock); 723 } else 724 BUG_ON(page_private(subdir) != freed); 725 } 726 offset = 0; 727 } 728done1: 729 shmem_dir_unmap(dir); 730done2: 731 if (inode->i_mapping->nrpages && (info->flags & SHMEM_PAGEIN)) { 732 /* 733 * Call truncate_inode_pages again: racing shmem_unuse_inode 734 * may have swizzled a page in from swap since 735 * truncate_pagecache or generic_delete_inode did it, before we 736 * lowered next_index. Also, though shmem_getpage checks 737 * i_size before adding to cache, no recheck after: so fix the 738 * narrow window there too. 739 */ 740 truncate_inode_pages_range(inode->i_mapping, start, end); 741 } 742 743 spin_lock(&info->lock); 744 info->flags &= ~SHMEM_TRUNCATE; 745 info->swapped -= nr_swaps_freed; 746 if (nr_pages_to_free) 747 shmem_free_blocks(inode, nr_pages_to_free); 748 shmem_recalc_inode(inode); 749 spin_unlock(&info->lock); 750 751 /* 752 * Empty swap vector directory pages to be freed? 753 */ 754 if (!list_empty(&pages_to_free)) { 755 pages_to_free.prev->next = NULL; 756 shmem_free_pages(pages_to_free.next); 757 } 758} 759EXPORT_SYMBOL_GPL(shmem_truncate_range); 760 761static int shmem_setattr(struct dentry *dentry, struct iattr *attr) 762{ 763 struct inode *inode = dentry->d_inode; 764 int error; 765 766 error = inode_change_ok(inode, attr); 767 if (error) 768 return error; 769 770 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 771 loff_t oldsize = inode->i_size; 772 loff_t newsize = attr->ia_size; 773 struct page *page = NULL; 774 775 if (newsize < oldsize) { 776 /* 777 * If truncating down to a partial page, then 778 * if that page is already allocated, hold it 779 * in memory until the truncation is over, so 780 * truncate_partial_page cannot miss it were 781 * it assigned to swap. 782 */ 783 if (newsize & (PAGE_CACHE_SIZE-1)) { 784 (void) shmem_getpage(inode, 785 newsize >> PAGE_CACHE_SHIFT, 786 &page, SGP_READ, NULL); 787 if (page) 788 unlock_page(page); 789 } 790 /* 791 * Reset SHMEM_PAGEIN flag so that shmem_truncate can 792 * detect if any pages might have been added to cache 793 * after truncate_inode_pages. But we needn't bother 794 * if it's being fully truncated to zero-length: the 795 * nrpages check is efficient enough in that case. 796 */ 797 if (newsize) { 798 struct shmem_inode_info *info = SHMEM_I(inode); 799 spin_lock(&info->lock); 800 info->flags &= ~SHMEM_PAGEIN; 801 spin_unlock(&info->lock); 802 } 803 } 804 if (newsize != oldsize) { 805 i_size_write(inode, newsize); 806 inode->i_ctime = inode->i_mtime = CURRENT_TIME; 807 } 808 if (newsize < oldsize) { 809 loff_t holebegin = round_up(newsize, PAGE_SIZE); 810 unmap_mapping_range(inode->i_mapping, holebegin, 0, 1); 811 shmem_truncate_range(inode, newsize, (loff_t)-1); 812 /* unmap again to remove racily COWed private pages */ 813 unmap_mapping_range(inode->i_mapping, holebegin, 0, 1); 814 } 815 if (page) 816 page_cache_release(page); 817 } 818 819 setattr_copy(inode, attr); 820#ifdef CONFIG_TMPFS_POSIX_ACL 821 if (attr->ia_valid & ATTR_MODE) 822 error = generic_acl_chmod(inode); 823#endif 824 return error; 825} 826 827static void shmem_evict_inode(struct inode *inode) 828{ 829 struct shmem_inode_info *info = SHMEM_I(inode); 830 struct shmem_xattr *xattr, *nxattr; 831 832 if (inode->i_mapping->a_ops == &shmem_aops) { 833 shmem_unacct_size(info->flags, inode->i_size); 834 inode->i_size = 0; 835 shmem_truncate_range(inode, 0, (loff_t)-1); 836 if (!list_empty(&info->swaplist)) { 837 mutex_lock(&shmem_swaplist_mutex); 838 list_del_init(&info->swaplist); 839 mutex_unlock(&shmem_swaplist_mutex); 840 } 841 } 842 843 list_for_each_entry_safe(xattr, nxattr, &info->xattr_list, list) { 844 kfree(xattr->name); 845 kfree(xattr); 846 } 847 BUG_ON(inode->i_blocks); 848 shmem_free_inode(inode->i_sb); 849 end_writeback(inode); 850} 851 852static inline int shmem_find_swp(swp_entry_t entry, swp_entry_t *dir, swp_entry_t *edir) 853{ 854 swp_entry_t *ptr; 855 856 for (ptr = dir; ptr < edir; ptr++) { 857 if (ptr->val == entry.val) 858 return ptr - dir; 859 } 860 return -1; 861} 862 863static int shmem_unuse_inode(struct shmem_inode_info *info, swp_entry_t entry, struct page *page) 864{ 865 struct address_space *mapping; 866 unsigned long idx; 867 unsigned long size; 868 unsigned long limit; 869 unsigned long stage; 870 struct page **dir; 871 struct page *subdir; 872 swp_entry_t *ptr; 873 int offset; 874 int error; 875 876 idx = 0; 877 ptr = info->i_direct; 878 spin_lock(&info->lock); 879 if (!info->swapped) { 880 list_del_init(&info->swaplist); 881 goto lost2; 882 } 883 limit = info->next_index; 884 size = limit; 885 if (size > SHMEM_NR_DIRECT) 886 size = SHMEM_NR_DIRECT; 887 offset = shmem_find_swp(entry, ptr, ptr+size); 888 if (offset >= 0) { 889 shmem_swp_balance_unmap(); 890 goto found; 891 } 892 if (!info->i_indirect) 893 goto lost2; 894 895 dir = shmem_dir_map(info->i_indirect); 896 stage = SHMEM_NR_DIRECT + ENTRIES_PER_PAGEPAGE/2; 897 898 for (idx = SHMEM_NR_DIRECT; idx < limit; idx += ENTRIES_PER_PAGE, dir++) { 899 if (unlikely(idx == stage)) { 900 shmem_dir_unmap(dir-1); 901 if (cond_resched_lock(&info->lock)) { 902 /* check it has not been truncated */ 903 if (limit > info->next_index) { 904 limit = info->next_index; 905 if (idx >= limit) 906 goto lost2; 907 } 908 } 909 dir = shmem_dir_map(info->i_indirect) + 910 ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE; 911 while (!*dir) { 912 dir++; 913 idx += ENTRIES_PER_PAGEPAGE; 914 if (idx >= limit) 915 goto lost1; 916 } 917 stage = idx + ENTRIES_PER_PAGEPAGE; 918 subdir = *dir; 919 shmem_dir_unmap(dir); 920 dir = shmem_dir_map(subdir); 921 } 922 subdir = *dir; 923 if (subdir && page_private(subdir)) { 924 ptr = shmem_swp_map(subdir); 925 size = limit - idx; 926 if (size > ENTRIES_PER_PAGE) 927 size = ENTRIES_PER_PAGE; 928 offset = shmem_find_swp(entry, ptr, ptr+size); 929 shmem_swp_unmap(ptr); 930 if (offset >= 0) { 931 shmem_dir_unmap(dir); 932 ptr = shmem_swp_map(subdir); 933 goto found; 934 } 935 } 936 } 937lost1: 938 shmem_dir_unmap(dir-1); 939lost2: 940 spin_unlock(&info->lock); 941 return 0; 942found: 943 idx += offset; 944 ptr += offset; 945 946 /* 947 * Move _head_ to start search for next from here. 948 * But be careful: shmem_evict_inode checks list_empty without taking 949 * mutex, and there's an instant in list_move_tail when info->swaplist 950 * would appear empty, if it were the only one on shmem_swaplist. We 951 * could avoid doing it if inode NULL; or use this minor optimization. 952 */ 953 if (shmem_swaplist.next != &info->swaplist) 954 list_move_tail(&shmem_swaplist, &info->swaplist); 955 956 /* 957 * We rely on shmem_swaplist_mutex, not only to protect the swaplist, 958 * but also to hold up shmem_evict_inode(): so inode cannot be freed 959 * beneath us (pagelock doesn't help until the page is in pagecache). 960 */ 961 mapping = info->vfs_inode.i_mapping; 962 error = add_to_page_cache_locked(page, mapping, idx, GFP_NOWAIT); 963 /* which does mem_cgroup_uncharge_cache_page on error */ 964 965 if (error == -EEXIST) { 966 struct page *filepage = find_get_page(mapping, idx); 967 error = 1; 968 if (filepage) { 969 /* 970 * There might be a more uptodate page coming down 971 * from a stacked writepage: forget our swappage if so. 972 */ 973 if (PageUptodate(filepage)) 974 error = 0; 975 page_cache_release(filepage); 976 } 977 } 978 if (!error) { 979 delete_from_swap_cache(page); 980 set_page_dirty(page); 981 info->flags |= SHMEM_PAGEIN; 982 shmem_swp_set(info, ptr, 0); 983 swap_free(entry); 984 error = 1; /* not an error, but entry was found */ 985 } 986 shmem_swp_unmap(ptr); 987 spin_unlock(&info->lock); 988 return error; 989} 990 991/* 992 * shmem_unuse() search for an eventually swapped out shmem page. 993 */ 994int shmem_unuse(swp_entry_t entry, struct page *page) 995{ 996 struct list_head *p, *next; 997 struct shmem_inode_info *info; 998 int found = 0; 999 int error; 1000 1001 /* 1002 * Charge page using GFP_KERNEL while we can wait, before taking 1003 * the shmem_swaplist_mutex which might hold up shmem_writepage(). 1004 * Charged back to the user (not to caller) when swap account is used. 1005 * add_to_page_cache() will be called with GFP_NOWAIT. 1006 */ 1007 error = mem_cgroup_cache_charge(page, current->mm, GFP_KERNEL); 1008 if (error) 1009 goto out; 1010 /* 1011 * Try to preload while we can wait, to not make a habit of 1012 * draining atomic reserves; but don't latch on to this cpu, 1013 * it's okay if sometimes we get rescheduled after this. 1014 */ 1015 error = radix_tree_preload(GFP_KERNEL); 1016 if (error) 1017 goto uncharge; 1018 radix_tree_preload_end(); 1019 1020 mutex_lock(&shmem_swaplist_mutex); 1021 list_for_each_safe(p, next, &shmem_swaplist) { 1022 info = list_entry(p, struct shmem_inode_info, swaplist); 1023 found = shmem_unuse_inode(info, entry, page); 1024 cond_resched(); 1025 if (found) 1026 break; 1027 } 1028 mutex_unlock(&shmem_swaplist_mutex); 1029 1030uncharge: 1031 if (!found) 1032 mem_cgroup_uncharge_cache_page(page); 1033 if (found < 0) 1034 error = found; 1035out: 1036 unlock_page(page); 1037 page_cache_release(page); 1038 return error; 1039} 1040 1041/* 1042 * Move the page from the page cache to the swap cache. 1043 */ 1044static int shmem_writepage(struct page *page, struct writeback_control *wbc) 1045{ 1046 struct shmem_inode_info *info; 1047 swp_entry_t *entry, swap; 1048 struct address_space *mapping; 1049 unsigned long index; 1050 struct inode *inode; 1051 1052 BUG_ON(!PageLocked(page)); 1053 mapping = page->mapping; 1054 index = page->index; 1055 inode = mapping->host; 1056 info = SHMEM_I(inode); 1057 if (info->flags & VM_LOCKED) 1058 goto redirty; 1059 if (!total_swap_pages) 1060 goto redirty; 1061 1062 /* 1063 * shmem_backing_dev_info's capabilities prevent regular writeback or 1064 * sync from ever calling shmem_writepage; but a stacking filesystem 1065 * may use the ->writepage of its underlying filesystem, in which case 1066 * tmpfs should write out to swap only in response to memory pressure, 1067 * and not for the writeback threads or sync. However, in those cases, 1068 * we do still want to check if there's a redundant swappage to be 1069 * discarded. 1070 */ 1071 if (wbc->for_reclaim) 1072 swap = get_swap_page(); 1073 else 1074 swap.val = 0; 1075 1076 /* 1077 * Add inode to shmem_unuse()'s list of swapped-out inodes, 1078 * if it's not already there. Do it now because we cannot take 1079 * mutex while holding spinlock, and must do so before the page 1080 * is moved to swap cache, when its pagelock no longer protects 1081 * the inode from eviction. But don't unlock the mutex until 1082 * we've taken the spinlock, because shmem_unuse_inode() will 1083 * prune a !swapped inode from the swaplist under both locks. 1084 */ 1085 if (swap.val) { 1086 mutex_lock(&shmem_swaplist_mutex); 1087 if (list_empty(&info->swaplist)) 1088 list_add_tail(&info->swaplist, &shmem_swaplist); 1089 } 1090 1091 spin_lock(&info->lock); 1092 if (swap.val) 1093 mutex_unlock(&shmem_swaplist_mutex); 1094 1095 if (index >= info->next_index) { 1096 BUG_ON(!(info->flags & SHMEM_TRUNCATE)); 1097 goto unlock; 1098 } 1099 entry = shmem_swp_entry(info, index, NULL); 1100 if (entry->val) { 1101 /* 1102 * The more uptodate page coming down from a stacked 1103 * writepage should replace our old swappage. 1104 */ 1105 free_swap_and_cache(*entry); 1106 shmem_swp_set(info, entry, 0); 1107 } 1108 shmem_recalc_inode(inode); 1109 1110 if (swap.val && add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) { 1111 delete_from_page_cache(page); 1112 shmem_swp_set(info, entry, swap.val); 1113 shmem_swp_unmap(entry); 1114 swap_shmem_alloc(swap); 1115 spin_unlock(&info->lock); 1116 BUG_ON(page_mapped(page)); 1117 swap_writepage(page, wbc); 1118 return 0; 1119 } 1120 1121 shmem_swp_unmap(entry); 1122unlock: 1123 spin_unlock(&info->lock); 1124 /* 1125 * add_to_swap_cache() doesn't return -EEXIST, so we can safely 1126 * clear SWAP_HAS_CACHE flag. 1127 */ 1128 swapcache_free(swap, NULL); 1129redirty: 1130 set_page_dirty(page); 1131 if (wbc->for_reclaim) 1132 return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */ 1133 unlock_page(page); 1134 return 0; 1135} 1136 1137#ifdef CONFIG_NUMA 1138#ifdef CONFIG_TMPFS 1139static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1140{ 1141 char buffer[64]; 1142 1143 if (!mpol || mpol->mode == MPOL_DEFAULT) 1144 return; /* show nothing */ 1145 1146 mpol_to_str(buffer, sizeof(buffer), mpol, 1); 1147 1148 seq_printf(seq, ",mpol=%s", buffer); 1149} 1150 1151static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1152{ 1153 struct mempolicy *mpol = NULL; 1154 if (sbinfo->mpol) { 1155 spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */ 1156 mpol = sbinfo->mpol; 1157 mpol_get(mpol); 1158 spin_unlock(&sbinfo->stat_lock); 1159 } 1160 return mpol; 1161} 1162#endif /* CONFIG_TMPFS */ 1163 1164static struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp, 1165 struct shmem_inode_info *info, unsigned long idx) 1166{ 1167 struct mempolicy mpol, *spol; 1168 struct vm_area_struct pvma; 1169 struct page *page; 1170 1171 spol = mpol_cond_copy(&mpol, 1172 mpol_shared_policy_lookup(&info->policy, idx)); 1173 1174 /* Create a pseudo vma that just contains the policy */ 1175 pvma.vm_start = 0; 1176 pvma.vm_pgoff = idx; 1177 pvma.vm_ops = NULL; 1178 pvma.vm_policy = spol; 1179 page = swapin_readahead(entry, gfp, &pvma, 0); 1180 return page; 1181} 1182 1183static struct page *shmem_alloc_page(gfp_t gfp, 1184 struct shmem_inode_info *info, unsigned long idx) 1185{ 1186 struct vm_area_struct pvma; 1187 1188 /* Create a pseudo vma that just contains the policy */ 1189 pvma.vm_start = 0; 1190 pvma.vm_pgoff = idx; 1191 pvma.vm_ops = NULL; 1192 pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx); 1193 1194 /* 1195 * alloc_page_vma() will drop the shared policy reference 1196 */ 1197 return alloc_page_vma(gfp, &pvma, 0); 1198} 1199#else /* !CONFIG_NUMA */ 1200#ifdef CONFIG_TMPFS 1201static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *p) 1202{ 1203} 1204#endif /* CONFIG_TMPFS */ 1205 1206static inline struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp, 1207 struct shmem_inode_info *info, unsigned long idx) 1208{ 1209 return swapin_readahead(entry, gfp, NULL, 0); 1210} 1211 1212static inline struct page *shmem_alloc_page(gfp_t gfp, 1213 struct shmem_inode_info *info, unsigned long idx) 1214{ 1215 return alloc_page(gfp); 1216} 1217#endif /* CONFIG_NUMA */ 1218 1219#if !defined(CONFIG_NUMA) || !defined(CONFIG_TMPFS) 1220static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1221{ 1222 return NULL; 1223} 1224#endif 1225 1226/* 1227 * shmem_getpage - either get the page from swap or allocate a new one 1228 * 1229 * If we allocate a new one we do not mark it dirty. That's up to the 1230 * vm. If we swap it in we mark it dirty since we also free the swap 1231 * entry since a page cannot live in both the swap and page cache 1232 */ 1233static int shmem_getpage(struct inode *inode, unsigned long idx, 1234 struct page **pagep, enum sgp_type sgp, int *type) 1235{ 1236 struct address_space *mapping = inode->i_mapping; 1237 struct shmem_inode_info *info = SHMEM_I(inode); 1238 struct shmem_sb_info *sbinfo; 1239 struct page *filepage = *pagep; 1240 struct page *swappage; 1241 struct page *prealloc_page = NULL; 1242 swp_entry_t *entry; 1243 swp_entry_t swap; 1244 gfp_t gfp; 1245 int error; 1246 1247 if (idx >= SHMEM_MAX_INDEX) 1248 return -EFBIG; 1249 1250 if (type) 1251 *type = 0; 1252 1253 /* 1254 * Normally, filepage is NULL on entry, and either found 1255 * uptodate immediately, or allocated and zeroed, or read 1256 * in under swappage, which is then assigned to filepage. 1257 * But shmem_readpage (required for splice) passes in a locked 1258 * filepage, which may be found not uptodate by other callers 1259 * too, and may need to be copied from the swappage read in. 1260 */ 1261repeat: 1262 if (!filepage) 1263 filepage = find_lock_page(mapping, idx); 1264 if (filepage && PageUptodate(filepage)) 1265 goto done; 1266 gfp = mapping_gfp_mask(mapping); 1267 if (!filepage) { 1268 /* 1269 * Try to preload while we can wait, to not make a habit of 1270 * draining atomic reserves; but don't latch on to this cpu. 1271 */ 1272 error = radix_tree_preload(gfp & ~__GFP_HIGHMEM); 1273 if (error) 1274 goto failed; 1275 radix_tree_preload_end(); 1276 if (sgp != SGP_READ && !prealloc_page) { 1277 /* We don't care if this fails */ 1278 prealloc_page = shmem_alloc_page(gfp, info, idx); 1279 if (prealloc_page) { 1280 if (mem_cgroup_cache_charge(prealloc_page, 1281 current->mm, GFP_KERNEL)) { 1282 page_cache_release(prealloc_page); 1283 prealloc_page = NULL; 1284 } 1285 } 1286 } 1287 } 1288 error = 0; 1289 1290 spin_lock(&info->lock); 1291 shmem_recalc_inode(inode); 1292 entry = shmem_swp_alloc(info, idx, sgp); 1293 if (IS_ERR(entry)) { 1294 spin_unlock(&info->lock); 1295 error = PTR_ERR(entry); 1296 goto failed; 1297 } 1298 swap = *entry; 1299 1300 if (swap.val) { 1301 /* Look it up and read it in.. */ 1302 swappage = lookup_swap_cache(swap); 1303 if (!swappage) { 1304 shmem_swp_unmap(entry); 1305 spin_unlock(&info->lock); 1306 /* here we actually do the io */ 1307 if (type) 1308 *type |= VM_FAULT_MAJOR; 1309 swappage = shmem_swapin(swap, gfp, info, idx); 1310 if (!swappage) { 1311 spin_lock(&info->lock); 1312 entry = shmem_swp_alloc(info, idx, sgp); 1313 if (IS_ERR(entry)) 1314 error = PTR_ERR(entry); 1315 else { 1316 if (entry->val == swap.val) 1317 error = -ENOMEM; 1318 shmem_swp_unmap(entry); 1319 } 1320 spin_unlock(&info->lock); 1321 if (error) 1322 goto failed; 1323 goto repeat; 1324 } 1325 wait_on_page_locked(swappage); 1326 page_cache_release(swappage); 1327 goto repeat; 1328 } 1329 1330 /* We have to do this with page locked to prevent races */ 1331 if (!trylock_page(swappage)) { 1332 shmem_swp_unmap(entry); 1333 spin_unlock(&info->lock); 1334 wait_on_page_locked(swappage); 1335 page_cache_release(swappage); 1336 goto repeat; 1337 } 1338 if (PageWriteback(swappage)) { 1339 shmem_swp_unmap(entry); 1340 spin_unlock(&info->lock); 1341 wait_on_page_writeback(swappage); 1342 unlock_page(swappage); 1343 page_cache_release(swappage); 1344 goto repeat; 1345 } 1346 if (!PageUptodate(swappage)) { 1347 shmem_swp_unmap(entry); 1348 spin_unlock(&info->lock); 1349 unlock_page(swappage); 1350 page_cache_release(swappage); 1351 error = -EIO; 1352 goto failed; 1353 } 1354 1355 if (filepage) { 1356 shmem_swp_set(info, entry, 0); 1357 shmem_swp_unmap(entry); 1358 delete_from_swap_cache(swappage); 1359 spin_unlock(&info->lock); 1360 copy_highpage(filepage, swappage); 1361 unlock_page(swappage); 1362 page_cache_release(swappage); 1363 flush_dcache_page(filepage); 1364 SetPageUptodate(filepage); 1365 set_page_dirty(filepage); 1366 swap_free(swap); 1367 } else if (!(error = add_to_page_cache_locked(swappage, mapping, 1368 idx, GFP_NOWAIT))) { 1369 info->flags |= SHMEM_PAGEIN; 1370 shmem_swp_set(info, entry, 0); 1371 shmem_swp_unmap(entry); 1372 delete_from_swap_cache(swappage); 1373 spin_unlock(&info->lock); 1374 filepage = swappage; 1375 set_page_dirty(filepage); 1376 swap_free(swap); 1377 } else { 1378 shmem_swp_unmap(entry); 1379 spin_unlock(&info->lock); 1380 if (error == -ENOMEM) { 1381 /* 1382 * reclaim from proper memory cgroup and 1383 * call memcg's OOM if needed. 1384 */ 1385 error = mem_cgroup_shmem_charge_fallback( 1386 swappage, 1387 current->mm, 1388 gfp); 1389 if (error) { 1390 unlock_page(swappage); 1391 page_cache_release(swappage); 1392 goto failed; 1393 } 1394 } 1395 unlock_page(swappage); 1396 page_cache_release(swappage); 1397 goto repeat; 1398 } 1399 } else if (sgp == SGP_READ && !filepage) { 1400 shmem_swp_unmap(entry); 1401 filepage = find_get_page(mapping, idx); 1402 if (filepage && 1403 (!PageUptodate(filepage) || !trylock_page(filepage))) { 1404 spin_unlock(&info->lock); 1405 wait_on_page_locked(filepage); 1406 page_cache_release(filepage); 1407 filepage = NULL; 1408 goto repeat; 1409 } 1410 spin_unlock(&info->lock); 1411 } else { 1412 shmem_swp_unmap(entry); 1413 sbinfo = SHMEM_SB(inode->i_sb); 1414 if (sbinfo->max_blocks) { 1415 if (percpu_counter_compare(&sbinfo->used_blocks, 1416 sbinfo->max_blocks) >= 0 || 1417 shmem_acct_block(info->flags)) 1418 goto nospace; 1419 percpu_counter_inc(&sbinfo->used_blocks); 1420 inode->i_blocks += BLOCKS_PER_PAGE; 1421 } else if (shmem_acct_block(info->flags)) 1422 goto nospace; 1423 1424 if (!filepage) { 1425 int ret; 1426 1427 if (!prealloc_page) { 1428 spin_unlock(&info->lock); 1429 filepage = shmem_alloc_page(gfp, info, idx); 1430 if (!filepage) { 1431 spin_lock(&info->lock); 1432 shmem_unacct_blocks(info->flags, 1); 1433 shmem_free_blocks(inode, 1); 1434 spin_unlock(&info->lock); 1435 error = -ENOMEM; 1436 goto failed; 1437 } 1438 SetPageSwapBacked(filepage); 1439 1440 /* 1441 * Precharge page while we can wait, compensate 1442 * after 1443 */ 1444 error = mem_cgroup_cache_charge(filepage, 1445 current->mm, GFP_KERNEL); 1446 if (error) { 1447 page_cache_release(filepage); 1448 spin_lock(&info->lock); 1449 shmem_unacct_blocks(info->flags, 1); 1450 shmem_free_blocks(inode, 1); 1451 spin_unlock(&info->lock); 1452 filepage = NULL; 1453 goto failed; 1454 } 1455 1456 spin_lock(&info->lock); 1457 } else { 1458 filepage = prealloc_page; 1459 prealloc_page = NULL; 1460 SetPageSwapBacked(filepage); 1461 } 1462 1463 entry = shmem_swp_alloc(info, idx, sgp); 1464 if (IS_ERR(entry)) 1465 error = PTR_ERR(entry); 1466 else { 1467 swap = *entry; 1468 shmem_swp_unmap(entry); 1469 } 1470 ret = error || swap.val; 1471 if (ret) 1472 mem_cgroup_uncharge_cache_page(filepage); 1473 else 1474 ret = add_to_page_cache_lru(filepage, mapping, 1475 idx, GFP_NOWAIT); 1476 /* 1477 * At add_to_page_cache_lru() failure, uncharge will 1478 * be done automatically. 1479 */ 1480 if (ret) { 1481 shmem_unacct_blocks(info->flags, 1); 1482 shmem_free_blocks(inode, 1); 1483 spin_unlock(&info->lock); 1484 page_cache_release(filepage); 1485 filepage = NULL; 1486 if (error) 1487 goto failed; 1488 goto repeat; 1489 } 1490 info->flags |= SHMEM_PAGEIN; 1491 } 1492 1493 info->alloced++; 1494 spin_unlock(&info->lock); 1495 clear_highpage(filepage); 1496 flush_dcache_page(filepage); 1497 SetPageUptodate(filepage); 1498 if (sgp == SGP_DIRTY) 1499 set_page_dirty(filepage); 1500 } 1501done: 1502 *pagep = filepage; 1503 error = 0; 1504 goto out; 1505 1506nospace: 1507 /* 1508 * Perhaps the page was brought in from swap between find_lock_page 1509 * and taking info->lock? We allow for that at add_to_page_cache_lru, 1510 * but must also avoid reporting a spurious ENOSPC while working on a 1511 * full tmpfs. (When filepage has been passed in to shmem_getpage, it 1512 * is already in page cache, which prevents this race from occurring.) 1513 */ 1514 if (!filepage) { 1515 struct page *page = find_get_page(mapping, idx); 1516 if (page) { 1517 spin_unlock(&info->lock); 1518 page_cache_release(page); 1519 goto repeat; 1520 } 1521 } 1522 spin_unlock(&info->lock); 1523 error = -ENOSPC; 1524failed: 1525 if (*pagep != filepage) { 1526 unlock_page(filepage); 1527 page_cache_release(filepage); 1528 } 1529out: 1530 if (prealloc_page) { 1531 mem_cgroup_uncharge_cache_page(prealloc_page); 1532 page_cache_release(prealloc_page); 1533 } 1534 return error; 1535} 1536 1537static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1538{ 1539 struct inode *inode = vma->vm_file->f_path.dentry->d_inode; 1540 int error; 1541 int ret; 1542 1543 if (((loff_t)vmf->pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode)) 1544 return VM_FAULT_SIGBUS; 1545 1546 error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret); 1547 if (error) 1548 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS); 1549 if (ret & VM_FAULT_MAJOR) { 1550 count_vm_event(PGMAJFAULT); 1551 mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT); 1552 } 1553 return ret | VM_FAULT_LOCKED; 1554} 1555 1556#ifdef CONFIG_NUMA 1557static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *new) 1558{ 1559 struct inode *i = vma->vm_file->f_path.dentry->d_inode; 1560 return mpol_set_shared_policy(&SHMEM_I(i)->policy, vma, new); 1561} 1562 1563static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, 1564 unsigned long addr) 1565{ 1566 struct inode *i = vma->vm_file->f_path.dentry->d_inode; 1567 unsigned long idx; 1568 1569 idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 1570 return mpol_shared_policy_lookup(&SHMEM_I(i)->policy, idx); 1571} 1572#endif 1573 1574int shmem_lock(struct file *file, int lock, struct user_struct *user) 1575{ 1576 struct inode *inode = file->f_path.dentry->d_inode; 1577 struct shmem_inode_info *info = SHMEM_I(inode); 1578 int retval = -ENOMEM; 1579 1580 spin_lock(&info->lock); 1581 if (lock && !(info->flags & VM_LOCKED)) { 1582 if (!user_shm_lock(inode->i_size, user)) 1583 goto out_nomem; 1584 info->flags |= VM_LOCKED; 1585 mapping_set_unevictable(file->f_mapping); 1586 } 1587 if (!lock && (info->flags & VM_LOCKED) && user) { 1588 user_shm_unlock(inode->i_size, user); 1589 info->flags &= ~VM_LOCKED; 1590 mapping_clear_unevictable(file->f_mapping); 1591 scan_mapping_unevictable_pages(file->f_mapping); 1592 } 1593 retval = 0; 1594 1595out_nomem: 1596 spin_unlock(&info->lock); 1597 return retval; 1598} 1599 1600static int shmem_mmap(struct file *file, struct vm_area_struct *vma) 1601{ 1602 file_accessed(file); 1603 vma->vm_ops = &shmem_vm_ops; 1604 vma->vm_flags |= VM_CAN_NONLINEAR; 1605 return 0; 1606} 1607 1608static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir, 1609 int mode, dev_t dev, unsigned long flags) 1610{ 1611 struct inode *inode; 1612 struct shmem_inode_info *info; 1613 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 1614 1615 if (shmem_reserve_inode(sb)) 1616 return NULL; 1617 1618 inode = new_inode(sb); 1619 if (inode) { 1620 inode->i_ino = get_next_ino(); 1621 inode_init_owner(inode, dir, mode); 1622 inode->i_blocks = 0; 1623 inode->i_mapping->backing_dev_info = &shmem_backing_dev_info; 1624 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME; 1625 inode->i_generation = get_seconds(); 1626 info = SHMEM_I(inode); 1627 memset(info, 0, (char *)inode - (char *)info); 1628 spin_lock_init(&info->lock); 1629 info->flags = flags & VM_NORESERVE; 1630 INIT_LIST_HEAD(&info->swaplist); 1631 INIT_LIST_HEAD(&info->xattr_list); 1632 cache_no_acl(inode); 1633 1634 switch (mode & S_IFMT) { 1635 default: 1636 inode->i_op = &shmem_special_inode_operations; 1637 init_special_inode(inode, mode, dev); 1638 break; 1639 case S_IFREG: 1640 inode->i_mapping->a_ops = &shmem_aops; 1641 inode->i_op = &shmem_inode_operations; 1642 inode->i_fop = &shmem_file_operations; 1643 mpol_shared_policy_init(&info->policy, 1644 shmem_get_sbmpol(sbinfo)); 1645 break; 1646 case S_IFDIR: 1647 inc_nlink(inode); 1648 /* Some things misbehave if size == 0 on a directory */ 1649 inode->i_size = 2 * BOGO_DIRENT_SIZE; 1650 inode->i_op = &shmem_dir_inode_operations; 1651 inode->i_fop = &simple_dir_operations; 1652 break; 1653 case S_IFLNK: 1654 /* 1655 * Must not load anything in the rbtree, 1656 * mpol_free_shared_policy will not be called. 1657 */ 1658 mpol_shared_policy_init(&info->policy, NULL); 1659 break; 1660 } 1661 } else 1662 shmem_free_inode(sb); 1663 return inode; 1664} 1665 1666#ifdef CONFIG_TMPFS 1667static const struct inode_operations shmem_symlink_inode_operations; 1668static const struct inode_operations shmem_symlink_inline_operations; 1669 1670/* 1671 * Normally tmpfs avoids the use of shmem_readpage and shmem_write_begin; 1672 * but providing them allows a tmpfs file to be used for splice, sendfile, and 1673 * below the loop driver, in the generic fashion that many filesystems support. 1674 */ 1675static int shmem_readpage(struct file *file, struct page *page) 1676{ 1677 struct inode *inode = page->mapping->host; 1678 int error = shmem_getpage(inode, page->index, &page, SGP_CACHE, NULL); 1679 unlock_page(page); 1680 return error; 1681} 1682 1683static int 1684shmem_write_begin(struct file *file, struct address_space *mapping, 1685 loff_t pos, unsigned len, unsigned flags, 1686 struct page **pagep, void **fsdata) 1687{ 1688 struct inode *inode = mapping->host; 1689 pgoff_t index = pos >> PAGE_CACHE_SHIFT; 1690 *pagep = NULL; 1691 return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL); 1692} 1693 1694static int 1695shmem_write_end(struct file *file, struct address_space *mapping, 1696 loff_t pos, unsigned len, unsigned copied, 1697 struct page *page, void *fsdata) 1698{ 1699 struct inode *inode = mapping->host; 1700 1701 if (pos + copied > inode->i_size) 1702 i_size_write(inode, pos + copied); 1703 1704 set_page_dirty(page); 1705 unlock_page(page); 1706 page_cache_release(page); 1707 1708 return copied; 1709} 1710 1711static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor) 1712{ 1713 struct inode *inode = filp->f_path.dentry->d_inode; 1714 struct address_space *mapping = inode->i_mapping; 1715 unsigned long index, offset; 1716 enum sgp_type sgp = SGP_READ; 1717 1718 /* 1719 * Might this read be for a stacking filesystem? Then when reading 1720 * holes of a sparse file, we actually need to allocate those pages, 1721 * and even mark them dirty, so it cannot exceed the max_blocks limit. 1722 */ 1723 if (segment_eq(get_fs(), KERNEL_DS)) 1724 sgp = SGP_DIRTY; 1725 1726 index = *ppos >> PAGE_CACHE_SHIFT; 1727 offset = *ppos & ~PAGE_CACHE_MASK; 1728 1729 for (;;) { 1730 struct page *page = NULL; 1731 unsigned long end_index, nr, ret; 1732 loff_t i_size = i_size_read(inode); 1733 1734 end_index = i_size >> PAGE_CACHE_SHIFT; 1735 if (index > end_index) 1736 break; 1737 if (index == end_index) { 1738 nr = i_size & ~PAGE_CACHE_MASK; 1739 if (nr <= offset) 1740 break; 1741 } 1742 1743 desc->error = shmem_getpage(inode, index, &page, sgp, NULL); 1744 if (desc->error) { 1745 if (desc->error == -EINVAL) 1746 desc->error = 0; 1747 break; 1748 } 1749 if (page) 1750 unlock_page(page); 1751 1752 /* 1753 * We must evaluate after, since reads (unlike writes) 1754 * are called without i_mutex protection against truncate 1755 */ 1756 nr = PAGE_CACHE_SIZE; 1757 i_size = i_size_read(inode); 1758 end_index = i_size >> PAGE_CACHE_SHIFT; 1759 if (index == end_index) { 1760 nr = i_size & ~PAGE_CACHE_MASK; 1761 if (nr <= offset) { 1762 if (page) 1763 page_cache_release(page); 1764 break; 1765 } 1766 } 1767 nr -= offset; 1768 1769 if (page) { 1770 /* 1771 * If users can be writing to this page using arbitrary 1772 * virtual addresses, take care about potential aliasing 1773 * before reading the page on the kernel side. 1774 */ 1775 if (mapping_writably_mapped(mapping)) 1776 flush_dcache_page(page); 1777 /* 1778 * Mark the page accessed if we read the beginning. 1779 */ 1780 if (!offset) 1781 mark_page_accessed(page); 1782 } else { 1783 page = ZERO_PAGE(0); 1784 page_cache_get(page); 1785 } 1786 1787 /* 1788 * Ok, we have the page, and it's up-to-date, so 1789 * now we can copy it to user space... 1790 * 1791 * The actor routine returns how many bytes were actually used.. 1792 * NOTE! This may not be the same as how much of a user buffer 1793 * we filled up (we may be padding etc), so we can only update 1794 * "pos" here (the actor routine has to update the user buffer 1795 * pointers and the remaining count). 1796 */ 1797 ret = actor(desc, page, offset, nr); 1798 offset += ret; 1799 index += offset >> PAGE_CACHE_SHIFT; 1800 offset &= ~PAGE_CACHE_MASK; 1801 1802 page_cache_release(page); 1803 if (ret != nr || !desc->count) 1804 break; 1805 1806 cond_resched(); 1807 } 1808 1809 *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset; 1810 file_accessed(filp); 1811} 1812 1813static ssize_t shmem_file_aio_read(struct kiocb *iocb, 1814 const struct iovec *iov, unsigned long nr_segs, loff_t pos) 1815{ 1816 struct file *filp = iocb->ki_filp; 1817 ssize_t retval; 1818 unsigned long seg; 1819 size_t count; 1820 loff_t *ppos = &iocb->ki_pos; 1821 1822 retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE); 1823 if (retval) 1824 return retval; 1825 1826 for (seg = 0; seg < nr_segs; seg++) { 1827 read_descriptor_t desc; 1828 1829 desc.written = 0; 1830 desc.arg.buf = iov[seg].iov_base; 1831 desc.count = iov[seg].iov_len; 1832 if (desc.count == 0) 1833 continue; 1834 desc.error = 0; 1835 do_shmem_file_read(filp, ppos, &desc, file_read_actor); 1836 retval += desc.written; 1837 if (desc.error) { 1838 retval = retval ?: desc.error; 1839 break; 1840 } 1841 if (desc.count > 0) 1842 break; 1843 } 1844 return retval; 1845} 1846 1847static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf) 1848{ 1849 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb); 1850 1851 buf->f_type = TMPFS_MAGIC; 1852 buf->f_bsize = PAGE_CACHE_SIZE; 1853 buf->f_namelen = NAME_MAX; 1854 if (sbinfo->max_blocks) { 1855 buf->f_blocks = sbinfo->max_blocks; 1856 buf->f_bavail = buf->f_bfree = 1857 sbinfo->max_blocks - percpu_counter_sum(&sbinfo->used_blocks); 1858 } 1859 if (sbinfo->max_inodes) { 1860 buf->f_files = sbinfo->max_inodes; 1861 buf->f_ffree = sbinfo->free_inodes; 1862 } 1863 /* else leave those fields 0 like simple_statfs */ 1864 return 0; 1865} 1866 1867/* 1868 * File creation. Allocate an inode, and we're done.. 1869 */ 1870static int 1871shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev) 1872{ 1873 struct inode *inode; 1874 int error = -ENOSPC; 1875 1876 inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE); 1877 if (inode) { 1878 error = security_inode_init_security(inode, dir, 1879 &dentry->d_name, NULL, 1880 NULL, NULL); 1881 if (error) { 1882 if (error != -EOPNOTSUPP) { 1883 iput(inode); 1884 return error; 1885 } 1886 } 1887#ifdef CONFIG_TMPFS_POSIX_ACL 1888 error = generic_acl_init(inode, dir); 1889 if (error) { 1890 iput(inode); 1891 return error; 1892 } 1893#else 1894 error = 0; 1895#endif 1896 dir->i_size += BOGO_DIRENT_SIZE; 1897 dir->i_ctime = dir->i_mtime = CURRENT_TIME; 1898 d_instantiate(dentry, inode); 1899 dget(dentry); /* Extra count - pin the dentry in core */ 1900 } 1901 return error; 1902} 1903 1904static int shmem_mkdir(struct inode *dir, struct dentry *dentry, int mode) 1905{ 1906 int error; 1907 1908 if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0))) 1909 return error; 1910 inc_nlink(dir); 1911 return 0; 1912} 1913 1914static int shmem_create(struct inode *dir, struct dentry *dentry, int mode, 1915 struct nameidata *nd) 1916{ 1917 return shmem_mknod(dir, dentry, mode | S_IFREG, 0); 1918} 1919 1920/* 1921 * Link a file.. 1922 */ 1923static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) 1924{ 1925 struct inode *inode = old_dentry->d_inode; 1926 int ret; 1927 1928 /* 1929 * No ordinary (disk based) filesystem counts links as inodes; 1930 * but each new link needs a new dentry, pinning lowmem, and 1931 * tmpfs dentries cannot be pruned until they are unlinked. 1932 */ 1933 ret = shmem_reserve_inode(inode->i_sb); 1934 if (ret) 1935 goto out; 1936 1937 dir->i_size += BOGO_DIRENT_SIZE; 1938 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME; 1939 inc_nlink(inode); 1940 ihold(inode); /* New dentry reference */ 1941 dget(dentry); /* Extra pinning count for the created dentry */ 1942 d_instantiate(dentry, inode); 1943out: 1944 return ret; 1945} 1946 1947static int shmem_unlink(struct inode *dir, struct dentry *dentry) 1948{ 1949 struct inode *inode = dentry->d_inode; 1950 1951 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode)) 1952 shmem_free_inode(inode->i_sb); 1953 1954 dir->i_size -= BOGO_DIRENT_SIZE; 1955 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME; 1956 drop_nlink(inode); 1957 dput(dentry); /* Undo the count from "create" - this does all the work */ 1958 return 0; 1959} 1960 1961static int shmem_rmdir(struct inode *dir, struct dentry *dentry) 1962{ 1963 if (!simple_empty(dentry)) 1964 return -ENOTEMPTY; 1965 1966 drop_nlink(dentry->d_inode); 1967 drop_nlink(dir); 1968 return shmem_unlink(dir, dentry); 1969} 1970 1971/* 1972 * The VFS layer already does all the dentry stuff for rename, 1973 * we just have to decrement the usage count for the target if 1974 * it exists so that the VFS layer correctly free's it when it 1975 * gets overwritten. 1976 */ 1977static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry) 1978{ 1979 struct inode *inode = old_dentry->d_inode; 1980 int they_are_dirs = S_ISDIR(inode->i_mode); 1981 1982 if (!simple_empty(new_dentry)) 1983 return -ENOTEMPTY; 1984 1985 if (new_dentry->d_inode) { 1986 (void) shmem_unlink(new_dir, new_dentry); 1987 if (they_are_dirs) 1988 drop_nlink(old_dir); 1989 } else if (they_are_dirs) { 1990 drop_nlink(old_dir); 1991 inc_nlink(new_dir); 1992 } 1993 1994 old_dir->i_size -= BOGO_DIRENT_SIZE; 1995 new_dir->i_size += BOGO_DIRENT_SIZE; 1996 old_dir->i_ctime = old_dir->i_mtime = 1997 new_dir->i_ctime = new_dir->i_mtime = 1998 inode->i_ctime = CURRENT_TIME; 1999 return 0; 2000} 2001 2002static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname) 2003{ 2004 int error; 2005 int len; 2006 struct inode *inode; 2007 struct page *page = NULL; 2008 char *kaddr; 2009 struct shmem_inode_info *info; 2010 2011 len = strlen(symname) + 1; 2012 if (len > PAGE_CACHE_SIZE) 2013 return -ENAMETOOLONG; 2014 2015 inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE); 2016 if (!inode) 2017 return -ENOSPC; 2018 2019 error = security_inode_init_security(inode, dir, &dentry->d_name, NULL, 2020 NULL, NULL); 2021 if (error) { 2022 if (error != -EOPNOTSUPP) { 2023 iput(inode); 2024 return error; 2025 } 2026 error = 0; 2027 } 2028 2029 info = SHMEM_I(inode); 2030 inode->i_size = len-1; 2031 if (len <= SHMEM_SYMLINK_INLINE_LEN) { 2032 /* do it inline */ 2033 memcpy(info->inline_symlink, symname, len); 2034 inode->i_op = &shmem_symlink_inline_operations; 2035 } else { 2036 error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL); 2037 if (error) { 2038 iput(inode); 2039 return error; 2040 } 2041 inode->i_mapping->a_ops = &shmem_aops; 2042 inode->i_op = &shmem_symlink_inode_operations; 2043 kaddr = kmap_atomic(page, KM_USER0); 2044 memcpy(kaddr, symname, len); 2045 kunmap_atomic(kaddr, KM_USER0); 2046 set_page_dirty(page); 2047 unlock_page(page); 2048 page_cache_release(page); 2049 } 2050 dir->i_size += BOGO_DIRENT_SIZE; 2051 dir->i_ctime = dir->i_mtime = CURRENT_TIME; 2052 d_instantiate(dentry, inode); 2053 dget(dentry); 2054 return 0; 2055} 2056 2057static void *shmem_follow_link_inline(struct dentry *dentry, struct nameidata *nd) 2058{ 2059 nd_set_link(nd, SHMEM_I(dentry->d_inode)->inline_symlink); 2060 return NULL; 2061} 2062 2063static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd) 2064{ 2065 struct page *page = NULL; 2066 int res = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL); 2067 nd_set_link(nd, res ? ERR_PTR(res) : kmap(page)); 2068 if (page) 2069 unlock_page(page); 2070 return page; 2071} 2072 2073static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie) 2074{ 2075 if (!IS_ERR(nd_get_link(nd))) { 2076 struct page *page = cookie; 2077 kunmap(page); 2078 mark_page_accessed(page); 2079 page_cache_release(page); 2080 } 2081} 2082 2083#ifdef CONFIG_TMPFS_XATTR 2084/* 2085 * Superblocks without xattr inode operations may get some security.* xattr 2086 * support from the LSM "for free". As soon as we have any other xattrs 2087 * like ACLs, we also need to implement the security.* handlers at 2088 * filesystem level, though. 2089 */ 2090 2091static int shmem_xattr_get(struct dentry *dentry, const char *name, 2092 void *buffer, size_t size) 2093{ 2094 struct shmem_inode_info *info; 2095 struct shmem_xattr *xattr; 2096 int ret = -ENODATA; 2097 2098 info = SHMEM_I(dentry->d_inode); 2099 2100 spin_lock(&info->lock); 2101 list_for_each_entry(xattr, &info->xattr_list, list) { 2102 if (strcmp(name, xattr->name)) 2103 continue; 2104 2105 ret = xattr->size; 2106 if (buffer) { 2107 if (size < xattr->size) 2108 ret = -ERANGE; 2109 else 2110 memcpy(buffer, xattr->value, xattr->size); 2111 } 2112 break; 2113 } 2114 spin_unlock(&info->lock); 2115 return ret; 2116} 2117 2118static int shmem_xattr_set(struct dentry *dentry, const char *name, 2119 const void *value, size_t size, int flags) 2120{ 2121 struct inode *inode = dentry->d_inode; 2122 struct shmem_inode_info *info = SHMEM_I(inode); 2123 struct shmem_xattr *xattr; 2124 struct shmem_xattr *new_xattr = NULL; 2125 size_t len; 2126 int err = 0; 2127 2128 /* value == NULL means remove */ 2129 if (value) { 2130 /* wrap around? */ 2131 len = sizeof(*new_xattr) + size; 2132 if (len <= sizeof(*new_xattr)) 2133 return -ENOMEM; 2134 2135 new_xattr = kmalloc(len, GFP_KERNEL); 2136 if (!new_xattr) 2137 return -ENOMEM; 2138 2139 new_xattr->name = kstrdup(name, GFP_KERNEL); 2140 if (!new_xattr->name) { 2141 kfree(new_xattr); 2142 return -ENOMEM; 2143 } 2144 2145 new_xattr->size = size; 2146 memcpy(new_xattr->value, value, size); 2147 } 2148 2149 spin_lock(&info->lock); 2150 list_for_each_entry(xattr, &info->xattr_list, list) { 2151 if (!strcmp(name, xattr->name)) { 2152 if (flags & XATTR_CREATE) { 2153 xattr = new_xattr; 2154 err = -EEXIST; 2155 } else if (new_xattr) { 2156 list_replace(&xattr->list, &new_xattr->list); 2157 } else { 2158 list_del(&xattr->list); 2159 } 2160 goto out; 2161 } 2162 } 2163 if (flags & XATTR_REPLACE) { 2164 xattr = new_xattr; 2165 err = -ENODATA; 2166 } else { 2167 list_add(&new_xattr->list, &info->xattr_list); 2168 xattr = NULL; 2169 } 2170out: 2171 spin_unlock(&info->lock); 2172 if (xattr) 2173 kfree(xattr->name); 2174 kfree(xattr); 2175 return err; 2176} 2177 2178 2179static const struct xattr_handler *shmem_xattr_handlers[] = { 2180#ifdef CONFIG_TMPFS_POSIX_ACL 2181 &generic_acl_access_handler, 2182 &generic_acl_default_handler, 2183#endif 2184 NULL 2185}; 2186 2187static int shmem_xattr_validate(const char *name) 2188{ 2189 struct { const char *prefix; size_t len; } arr[] = { 2190 { XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN }, 2191 { XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN } 2192 }; 2193 int i; 2194 2195 for (i = 0; i < ARRAY_SIZE(arr); i++) { 2196 size_t preflen = arr[i].len; 2197 if (strncmp(name, arr[i].prefix, preflen) == 0) { 2198 if (!name[preflen]) 2199 return -EINVAL; 2200 return 0; 2201 } 2202 } 2203 return -EOPNOTSUPP; 2204} 2205 2206static ssize_t shmem_getxattr(struct dentry *dentry, const char *name, 2207 void *buffer, size_t size) 2208{ 2209 int err; 2210 2211 /* 2212 * If this is a request for a synthetic attribute in the system.* 2213 * namespace use the generic infrastructure to resolve a handler 2214 * for it via sb->s_xattr. 2215 */ 2216 if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN)) 2217 return generic_getxattr(dentry, name, buffer, size); 2218 2219 err = shmem_xattr_validate(name); 2220 if (err) 2221 return err; 2222 2223 return shmem_xattr_get(dentry, name, buffer, size); 2224} 2225 2226static int shmem_setxattr(struct dentry *dentry, const char *name, 2227 const void *value, size_t size, int flags) 2228{ 2229 int err; 2230 2231 /* 2232 * If this is a request for a synthetic attribute in the system.* 2233 * namespace use the generic infrastructure to resolve a handler 2234 * for it via sb->s_xattr. 2235 */ 2236 if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN)) 2237 return generic_setxattr(dentry, name, value, size, flags); 2238 2239 err = shmem_xattr_validate(name); 2240 if (err) 2241 return err; 2242 2243 if (size == 0) 2244 value = ""; /* empty EA, do not remove */ 2245 2246 return shmem_xattr_set(dentry, name, value, size, flags); 2247 2248} 2249 2250static int shmem_removexattr(struct dentry *dentry, const char *name) 2251{ 2252 int err; 2253 2254 /* 2255 * If this is a request for a synthetic attribute in the system.* 2256 * namespace use the generic infrastructure to resolve a handler 2257 * for it via sb->s_xattr. 2258 */ 2259 if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN)) 2260 return generic_removexattr(dentry, name); 2261 2262 err = shmem_xattr_validate(name); 2263 if (err) 2264 return err; 2265 2266 return shmem_xattr_set(dentry, name, NULL, 0, XATTR_REPLACE); 2267} 2268 2269static bool xattr_is_trusted(const char *name) 2270{ 2271 return !strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN); 2272} 2273 2274static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size) 2275{ 2276 bool trusted = capable(CAP_SYS_ADMIN); 2277 struct shmem_xattr *xattr; 2278 struct shmem_inode_info *info; 2279 size_t used = 0; 2280 2281 info = SHMEM_I(dentry->d_inode); 2282 2283 spin_lock(&info->lock); 2284 list_for_each_entry(xattr, &info->xattr_list, list) { 2285 size_t len; 2286 2287 /* skip "trusted." attributes for unprivileged callers */ 2288 if (!trusted && xattr_is_trusted(xattr->name)) 2289 continue; 2290 2291 len = strlen(xattr->name) + 1; 2292 used += len; 2293 if (buffer) { 2294 if (size < used) { 2295 used = -ERANGE; 2296 break; 2297 } 2298 memcpy(buffer, xattr->name, len); 2299 buffer += len; 2300 } 2301 } 2302 spin_unlock(&info->lock); 2303 2304 return used; 2305} 2306#endif /* CONFIG_TMPFS_XATTR */ 2307 2308static const struct inode_operations shmem_symlink_inline_operations = { 2309 .readlink = generic_readlink, 2310 .follow_link = shmem_follow_link_inline, 2311#ifdef CONFIG_TMPFS_XATTR 2312 .setxattr = shmem_setxattr, 2313 .getxattr = shmem_getxattr, 2314 .listxattr = shmem_listxattr, 2315 .removexattr = shmem_removexattr, 2316#endif 2317}; 2318 2319static const struct inode_operations shmem_symlink_inode_operations = { 2320 .readlink = generic_readlink, 2321 .follow_link = shmem_follow_link, 2322 .put_link = shmem_put_link, 2323#ifdef CONFIG_TMPFS_XATTR 2324 .setxattr = shmem_setxattr, 2325 .getxattr = shmem_getxattr, 2326 .listxattr = shmem_listxattr, 2327 .removexattr = shmem_removexattr, 2328#endif 2329}; 2330 2331static struct dentry *shmem_get_parent(struct dentry *child) 2332{ 2333 return ERR_PTR(-ESTALE); 2334} 2335 2336static int shmem_match(struct inode *ino, void *vfh) 2337{ 2338 __u32 *fh = vfh; 2339 __u64 inum = fh[2]; 2340 inum = (inum << 32) | fh[1]; 2341 return ino->i_ino == inum && fh[0] == ino->i_generation; 2342} 2343 2344static struct dentry *shmem_fh_to_dentry(struct super_block *sb, 2345 struct fid *fid, int fh_len, int fh_type) 2346{ 2347 struct inode *inode; 2348 struct dentry *dentry = NULL; 2349 u64 inum = fid->raw[2]; 2350 inum = (inum << 32) | fid->raw[1]; 2351 2352 if (fh_len < 3) 2353 return NULL; 2354 2355 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]), 2356 shmem_match, fid->raw); 2357 if (inode) { 2358 dentry = d_find_alias(inode); 2359 iput(inode); 2360 } 2361 2362 return dentry; 2363} 2364 2365static int shmem_encode_fh(struct dentry *dentry, __u32 *fh, int *len, 2366 int connectable) 2367{ 2368 struct inode *inode = dentry->d_inode; 2369 2370 if (*len < 3) { 2371 *len = 3; 2372 return 255; 2373 } 2374 2375 if (inode_unhashed(inode)) { 2376 /* Unfortunately insert_inode_hash is not idempotent, 2377 * so as we hash inodes here rather than at creation 2378 * time, we need a lock to ensure we only try 2379 * to do it once 2380 */ 2381 static DEFINE_SPINLOCK(lock); 2382 spin_lock(&lock); 2383 if (inode_unhashed(inode)) 2384 __insert_inode_hash(inode, 2385 inode->i_ino + inode->i_generation); 2386 spin_unlock(&lock); 2387 } 2388 2389 fh[0] = inode->i_generation; 2390 fh[1] = inode->i_ino; 2391 fh[2] = ((__u64)inode->i_ino) >> 32; 2392 2393 *len = 3; 2394 return 1; 2395} 2396 2397static const struct export_operations shmem_export_ops = { 2398 .get_parent = shmem_get_parent, 2399 .encode_fh = shmem_encode_fh, 2400 .fh_to_dentry = shmem_fh_to_dentry, 2401}; 2402 2403static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo, 2404 bool remount) 2405{ 2406 char *this_char, *value, *rest; 2407 2408 while (options != NULL) { 2409 this_char = options; 2410 for (;;) { 2411 /* 2412 * NUL-terminate this option: unfortunately, 2413 * mount options form a comma-separated list, 2414 * but mpol's nodelist may also contain commas. 2415 */ 2416 options = strchr(options, ','); 2417 if (options == NULL) 2418 break; 2419 options++; 2420 if (!isdigit(*options)) { 2421 options[-1] = '\0'; 2422 break; 2423 } 2424 } 2425 if (!*this_char) 2426 continue; 2427 if ((value = strchr(this_char,'=')) != NULL) { 2428 *value++ = 0; 2429 } else { 2430 printk(KERN_ERR 2431 "tmpfs: No value for mount option '%s'\n", 2432 this_char); 2433 return 1; 2434 } 2435 2436 if (!strcmp(this_char,"size")) { 2437 unsigned long long size; 2438 size = memparse(value,&rest); 2439 if (*rest == '%') { 2440 size <<= PAGE_SHIFT; 2441 size *= totalram_pages; 2442 do_div(size, 100); 2443 rest++; 2444 } 2445 if (*rest) 2446 goto bad_val; 2447 sbinfo->max_blocks = 2448 DIV_ROUND_UP(size, PAGE_CACHE_SIZE); 2449 } else if (!strcmp(this_char,"nr_blocks")) { 2450 sbinfo->max_blocks = memparse(value, &rest); 2451 if (*rest) 2452 goto bad_val; 2453 } else if (!strcmp(this_char,"nr_inodes")) { 2454 sbinfo->max_inodes = memparse(value, &rest); 2455 if (*rest) 2456 goto bad_val; 2457 } else if (!strcmp(this_char,"mode")) { 2458 if (remount) 2459 continue; 2460 sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777; 2461 if (*rest) 2462 goto bad_val; 2463 } else if (!strcmp(this_char,"uid")) { 2464 if (remount) 2465 continue; 2466 sbinfo->uid = simple_strtoul(value, &rest, 0); 2467 if (*rest) 2468 goto bad_val; 2469 } else if (!strcmp(this_char,"gid")) { 2470 if (remount) 2471 continue; 2472 sbinfo->gid = simple_strtoul(value, &rest, 0); 2473 if (*rest) 2474 goto bad_val; 2475 } else if (!strcmp(this_char,"mpol")) { 2476 if (mpol_parse_str(value, &sbinfo->mpol, 1)) 2477 goto bad_val; 2478 } else { 2479 printk(KERN_ERR "tmpfs: Bad mount option %s\n", 2480 this_char); 2481 return 1; 2482 } 2483 } 2484 return 0; 2485 2486bad_val: 2487 printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n", 2488 value, this_char); 2489 return 1; 2490 2491} 2492 2493static int shmem_remount_fs(struct super_block *sb, int *flags, char *data) 2494{ 2495 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 2496 struct shmem_sb_info config = *sbinfo; 2497 unsigned long inodes; 2498 int error = -EINVAL; 2499 2500 if (shmem_parse_options(data, &config, true)) 2501 return error; 2502 2503 spin_lock(&sbinfo->stat_lock); 2504 inodes = sbinfo->max_inodes - sbinfo->free_inodes; 2505 if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0) 2506 goto out; 2507 if (config.max_inodes < inodes) 2508 goto out; 2509 /* 2510 * Those tests also disallow limited->unlimited while any are in 2511 * use, so i_blocks will always be zero when max_blocks is zero; 2512 * but we must separately disallow unlimited->limited, because 2513 * in that case we have no record of how much is already in use. 2514 */ 2515 if (config.max_blocks && !sbinfo->max_blocks) 2516 goto out; 2517 if (config.max_inodes && !sbinfo->max_inodes) 2518 goto out; 2519 2520 error = 0; 2521 sbinfo->max_blocks = config.max_blocks; 2522 sbinfo->max_inodes = config.max_inodes; 2523 sbinfo->free_inodes = config.max_inodes - inodes; 2524 2525 mpol_put(sbinfo->mpol); 2526 sbinfo->mpol = config.mpol; /* transfers initial ref */ 2527out: 2528 spin_unlock(&sbinfo->stat_lock); 2529 return error; 2530} 2531 2532static int shmem_show_options(struct seq_file *seq, struct vfsmount *vfs) 2533{ 2534 struct shmem_sb_info *sbinfo = SHMEM_SB(vfs->mnt_sb); 2535 2536 if (sbinfo->max_blocks != shmem_default_max_blocks()) 2537 seq_printf(seq, ",size=%luk", 2538 sbinfo->max_blocks << (PAGE_CACHE_SHIFT - 10)); 2539 if (sbinfo->max_inodes != shmem_default_max_inodes()) 2540 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes); 2541 if (sbinfo->mode != (S_IRWXUGO | S_ISVTX)) 2542 seq_printf(seq, ",mode=%03o", sbinfo->mode); 2543 if (sbinfo->uid != 0) 2544 seq_printf(seq, ",uid=%u", sbinfo->uid); 2545 if (sbinfo->gid != 0) 2546 seq_printf(seq, ",gid=%u", sbinfo->gid); 2547 shmem_show_mpol(seq, sbinfo->mpol); 2548 return 0; 2549} 2550#endif /* CONFIG_TMPFS */ 2551 2552static void shmem_put_super(struct super_block *sb) 2553{ 2554 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 2555 2556 percpu_counter_destroy(&sbinfo->used_blocks); 2557 kfree(sbinfo); 2558 sb->s_fs_info = NULL; 2559} 2560 2561int shmem_fill_super(struct super_block *sb, void *data, int silent) 2562{ 2563 struct inode *inode; 2564 struct dentry *root; 2565 struct shmem_sb_info *sbinfo; 2566 int err = -ENOMEM; 2567 2568 /* Round up to L1_CACHE_BYTES to resist false sharing */ 2569 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info), 2570 L1_CACHE_BYTES), GFP_KERNEL); 2571 if (!sbinfo) 2572 return -ENOMEM; 2573 2574 sbinfo->mode = S_IRWXUGO | S_ISVTX; 2575 sbinfo->uid = current_fsuid(); 2576 sbinfo->gid = current_fsgid(); 2577 sb->s_fs_info = sbinfo; 2578 2579#ifdef CONFIG_TMPFS 2580 /* 2581 * Per default we only allow half of the physical ram per 2582 * tmpfs instance, limiting inodes to one per page of lowmem; 2583 * but the internal instance is left unlimited. 2584 */ 2585 if (!(sb->s_flags & MS_NOUSER)) { 2586 sbinfo->max_blocks = shmem_default_max_blocks(); 2587 sbinfo->max_inodes = shmem_default_max_inodes(); 2588 if (shmem_parse_options(data, sbinfo, false)) { 2589 err = -EINVAL; 2590 goto failed; 2591 } 2592 } 2593 sb->s_export_op = &shmem_export_ops; 2594#else 2595 sb->s_flags |= MS_NOUSER; 2596#endif 2597 2598 spin_lock_init(&sbinfo->stat_lock); 2599 if (percpu_counter_init(&sbinfo->used_blocks, 0)) 2600 goto failed; 2601 sbinfo->free_inodes = sbinfo->max_inodes; 2602 2603 sb->s_maxbytes = SHMEM_MAX_BYTES; 2604 sb->s_blocksize = PAGE_CACHE_SIZE; 2605 sb->s_blocksize_bits = PAGE_CACHE_SHIFT; 2606 sb->s_magic = TMPFS_MAGIC; 2607 sb->s_op = &shmem_ops; 2608 sb->s_time_gran = 1; 2609#ifdef CONFIG_TMPFS_XATTR 2610 sb->s_xattr = shmem_xattr_handlers; 2611#endif 2612#ifdef CONFIG_TMPFS_POSIX_ACL 2613 sb->s_flags |= MS_POSIXACL; 2614#endif 2615 2616 inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE); 2617 if (!inode) 2618 goto failed; 2619 inode->i_uid = sbinfo->uid; 2620 inode->i_gid = sbinfo->gid; 2621 root = d_alloc_root(inode); 2622 if (!root) 2623 goto failed_iput; 2624 sb->s_root = root; 2625 return 0; 2626 2627failed_iput: 2628 iput(inode); 2629failed: 2630 shmem_put_super(sb); 2631 return err; 2632} 2633 2634static struct kmem_cache *shmem_inode_cachep; 2635 2636static struct inode *shmem_alloc_inode(struct super_block *sb) 2637{ 2638 struct shmem_inode_info *p; 2639 p = (struct shmem_inode_info *)kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL); 2640 if (!p) 2641 return NULL; 2642 return &p->vfs_inode; 2643} 2644 2645static void shmem_i_callback(struct rcu_head *head) 2646{ 2647 struct inode *inode = container_of(head, struct inode, i_rcu); 2648 INIT_LIST_HEAD(&inode->i_dentry); 2649 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode)); 2650} 2651 2652static void shmem_destroy_inode(struct inode *inode) 2653{ 2654 if ((inode->i_mode & S_IFMT) == S_IFREG) { 2655 /* only struct inode is valid if it's an inline symlink */ 2656 mpol_free_shared_policy(&SHMEM_I(inode)->policy); 2657 } 2658 call_rcu(&inode->i_rcu, shmem_i_callback); 2659} 2660 2661static void init_once(void *foo) 2662{ 2663 struct shmem_inode_info *p = (struct shmem_inode_info *) foo; 2664 2665 inode_init_once(&p->vfs_inode); 2666} 2667 2668static int init_inodecache(void) 2669{ 2670 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache", 2671 sizeof(struct shmem_inode_info), 2672 0, SLAB_PANIC, init_once); 2673 return 0; 2674} 2675 2676static void destroy_inodecache(void) 2677{ 2678 kmem_cache_destroy(shmem_inode_cachep); 2679} 2680 2681static const struct address_space_operations shmem_aops = { 2682 .writepage = shmem_writepage, 2683 .set_page_dirty = __set_page_dirty_no_writeback, 2684#ifdef CONFIG_TMPFS 2685 .readpage = shmem_readpage, 2686 .write_begin = shmem_write_begin, 2687 .write_end = shmem_write_end, 2688#endif 2689 .migratepage = migrate_page, 2690 .error_remove_page = generic_error_remove_page, 2691}; 2692 2693static const struct file_operations shmem_file_operations = { 2694 .mmap = shmem_mmap, 2695#ifdef CONFIG_TMPFS 2696 .llseek = generic_file_llseek, 2697 .read = do_sync_read, 2698 .write = do_sync_write, 2699 .aio_read = shmem_file_aio_read, 2700 .aio_write = generic_file_aio_write, 2701 .fsync = noop_fsync, 2702 .splice_read = generic_file_splice_read, 2703 .splice_write = generic_file_splice_write, 2704#endif 2705}; 2706 2707static const struct inode_operations shmem_inode_operations = { 2708 .setattr = shmem_setattr, 2709 .truncate_range = shmem_truncate_range, 2710#ifdef CONFIG_TMPFS_XATTR 2711 .setxattr = shmem_setxattr, 2712 .getxattr = shmem_getxattr, 2713 .listxattr = shmem_listxattr, 2714 .removexattr = shmem_removexattr, 2715#endif 2716#ifdef CONFIG_TMPFS_POSIX_ACL 2717 .check_acl = generic_check_acl, 2718#endif 2719 2720}; 2721 2722static const struct inode_operations shmem_dir_inode_operations = { 2723#ifdef CONFIG_TMPFS 2724 .create = shmem_create, 2725 .lookup = simple_lookup, 2726 .link = shmem_link, 2727 .unlink = shmem_unlink, 2728 .symlink = shmem_symlink, 2729 .mkdir = shmem_mkdir, 2730 .rmdir = shmem_rmdir, 2731 .mknod = shmem_mknod, 2732 .rename = shmem_rename, 2733#endif 2734#ifdef CONFIG_TMPFS_XATTR 2735 .setxattr = shmem_setxattr, 2736 .getxattr = shmem_getxattr, 2737 .listxattr = shmem_listxattr, 2738 .removexattr = shmem_removexattr, 2739#endif 2740#ifdef CONFIG_TMPFS_POSIX_ACL 2741 .setattr = shmem_setattr, 2742 .check_acl = generic_check_acl, 2743#endif 2744}; 2745 2746static const struct inode_operations shmem_special_inode_operations = { 2747#ifdef CONFIG_TMPFS_XATTR 2748 .setxattr = shmem_setxattr, 2749 .getxattr = shmem_getxattr, 2750 .listxattr = shmem_listxattr, 2751 .removexattr = shmem_removexattr, 2752#endif 2753#ifdef CONFIG_TMPFS_POSIX_ACL 2754 .setattr = shmem_setattr, 2755 .check_acl = generic_check_acl, 2756#endif 2757}; 2758 2759static const struct super_operations shmem_ops = { 2760 .alloc_inode = shmem_alloc_inode, 2761 .destroy_inode = shmem_destroy_inode, 2762#ifdef CONFIG_TMPFS 2763 .statfs = shmem_statfs, 2764 .remount_fs = shmem_remount_fs, 2765 .show_options = shmem_show_options, 2766#endif 2767 .evict_inode = shmem_evict_inode, 2768 .drop_inode = generic_delete_inode, 2769 .put_super = shmem_put_super, 2770}; 2771 2772static const struct vm_operations_struct shmem_vm_ops = { 2773 .fault = shmem_fault, 2774#ifdef CONFIG_NUMA 2775 .set_policy = shmem_set_policy, 2776 .get_policy = shmem_get_policy, 2777#endif 2778}; 2779 2780 2781static struct dentry *shmem_mount(struct file_system_type *fs_type, 2782 int flags, const char *dev_name, void *data) 2783{ 2784 return mount_nodev(fs_type, flags, data, shmem_fill_super); 2785} 2786 2787static struct file_system_type tmpfs_fs_type = { 2788 .owner = THIS_MODULE, 2789 .name = "tmpfs", 2790 .mount = shmem_mount, 2791 .kill_sb = kill_litter_super, 2792}; 2793 2794int __init init_tmpfs(void) 2795{ 2796 int error; 2797 2798 error = bdi_init(&shmem_backing_dev_info); 2799 if (error) 2800 goto out4; 2801 2802 error = init_inodecache(); 2803 if (error) 2804 goto out3; 2805 2806 error = register_filesystem(&tmpfs_fs_type); 2807 if (error) { 2808 printk(KERN_ERR "Could not register tmpfs\n"); 2809 goto out2; 2810 } 2811 2812 shm_mnt = vfs_kern_mount(&tmpfs_fs_type, MS_NOUSER, 2813 tmpfs_fs_type.name, NULL); 2814 if (IS_ERR(shm_mnt)) { 2815 error = PTR_ERR(shm_mnt); 2816 printk(KERN_ERR "Could not kern_mount tmpfs\n"); 2817 goto out1; 2818 } 2819 return 0; 2820 2821out1: 2822 unregister_filesystem(&tmpfs_fs_type); 2823out2: 2824 destroy_inodecache(); 2825out3: 2826 bdi_destroy(&shmem_backing_dev_info); 2827out4: 2828 shm_mnt = ERR_PTR(error); 2829 return error; 2830} 2831 2832#ifdef CONFIG_CGROUP_MEM_RES_CTLR 2833/** 2834 * mem_cgroup_get_shmem_target - find a page or entry assigned to the shmem file 2835 * @inode: the inode to be searched 2836 * @pgoff: the offset to be searched 2837 * @pagep: the pointer for the found page to be stored 2838 * @ent: the pointer for the found swap entry to be stored 2839 * 2840 * If a page is found, refcount of it is incremented. Callers should handle 2841 * these refcount. 2842 */ 2843void mem_cgroup_get_shmem_target(struct inode *inode, pgoff_t pgoff, 2844 struct page **pagep, swp_entry_t *ent) 2845{ 2846 swp_entry_t entry = { .val = 0 }, *ptr; 2847 struct page *page = NULL; 2848 struct shmem_inode_info *info = SHMEM_I(inode); 2849 2850 if ((pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode)) 2851 goto out; 2852 2853 spin_lock(&info->lock); 2854 ptr = shmem_swp_entry(info, pgoff, NULL); 2855#ifdef CONFIG_SWAP 2856 if (ptr && ptr->val) { 2857 entry.val = ptr->val; 2858 page = find_get_page(&swapper_space, entry.val); 2859 } else 2860#endif 2861 page = find_get_page(inode->i_mapping, pgoff); 2862 if (ptr) 2863 shmem_swp_unmap(ptr); 2864 spin_unlock(&info->lock); 2865out: 2866 *pagep = page; 2867 *ent = entry; 2868} 2869#endif 2870 2871#else /* !CONFIG_SHMEM */ 2872 2873/* 2874 * tiny-shmem: simple shmemfs and tmpfs using ramfs code 2875 * 2876 * This is intended for small system where the benefits of the full 2877 * shmem code (swap-backed and resource-limited) are outweighed by 2878 * their complexity. On systems without swap this code should be 2879 * effectively equivalent, but much lighter weight. 2880 */ 2881 2882#include <linux/ramfs.h> 2883 2884static struct file_system_type tmpfs_fs_type = { 2885 .name = "tmpfs", 2886 .mount = ramfs_mount, 2887 .kill_sb = kill_litter_super, 2888}; 2889 2890int __init init_tmpfs(void) 2891{ 2892 BUG_ON(register_filesystem(&tmpfs_fs_type) != 0); 2893 2894 shm_mnt = kern_mount(&tmpfs_fs_type); 2895 BUG_ON(IS_ERR(shm_mnt)); 2896 2897 return 0; 2898} 2899 2900int shmem_unuse(swp_entry_t entry, struct page *page) 2901{ 2902 return 0; 2903} 2904 2905int shmem_lock(struct file *file, int lock, struct user_struct *user) 2906{ 2907 return 0; 2908} 2909 2910void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end) 2911{ 2912 truncate_inode_pages_range(inode->i_mapping, start, end); 2913} 2914EXPORT_SYMBOL_GPL(shmem_truncate_range); 2915 2916#ifdef CONFIG_CGROUP_MEM_RES_CTLR 2917/** 2918 * mem_cgroup_get_shmem_target - find a page or entry assigned to the shmem file 2919 * @inode: the inode to be searched 2920 * @pgoff: the offset to be searched 2921 * @pagep: the pointer for the found page to be stored 2922 * @ent: the pointer for the found swap entry to be stored 2923 * 2924 * If a page is found, refcount of it is incremented. Callers should handle 2925 * these refcount. 2926 */ 2927void mem_cgroup_get_shmem_target(struct inode *inode, pgoff_t pgoff, 2928 struct page **pagep, swp_entry_t *ent) 2929{ 2930 struct page *page = NULL; 2931 2932 if ((pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode)) 2933 goto out; 2934 page = find_get_page(inode->i_mapping, pgoff); 2935out: 2936 *pagep = page; 2937 *ent = (swp_entry_t){ .val = 0 }; 2938} 2939#endif 2940 2941#define shmem_vm_ops generic_file_vm_ops 2942#define shmem_file_operations ramfs_file_operations 2943#define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev) 2944#define shmem_acct_size(flags, size) 0 2945#define shmem_unacct_size(flags, size) do {} while (0) 2946#define SHMEM_MAX_BYTES MAX_LFS_FILESIZE 2947 2948#endif /* CONFIG_SHMEM */ 2949 2950/* common code */ 2951 2952/** 2953 * shmem_file_setup - get an unlinked file living in tmpfs 2954 * @name: name for dentry (to be seen in /proc/<pid>/maps 2955 * @size: size to be set for the file 2956 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 2957 */ 2958struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags) 2959{ 2960 int error; 2961 struct file *file; 2962 struct inode *inode; 2963 struct path path; 2964 struct dentry *root; 2965 struct qstr this; 2966 2967 if (IS_ERR(shm_mnt)) 2968 return (void *)shm_mnt; 2969 2970 if (size < 0 || size > SHMEM_MAX_BYTES) 2971 return ERR_PTR(-EINVAL); 2972 2973 if (shmem_acct_size(flags, size)) 2974 return ERR_PTR(-ENOMEM); 2975 2976 error = -ENOMEM; 2977 this.name = name; 2978 this.len = strlen(name); 2979 this.hash = 0; /* will go */ 2980 root = shm_mnt->mnt_root; 2981 path.dentry = d_alloc(root, &this); 2982 if (!path.dentry) 2983 goto put_memory; 2984 path.mnt = mntget(shm_mnt); 2985 2986 error = -ENOSPC; 2987 inode = shmem_get_inode(root->d_sb, NULL, S_IFREG | S_IRWXUGO, 0, flags); 2988 if (!inode) 2989 goto put_dentry; 2990 2991 d_instantiate(path.dentry, inode); 2992 inode->i_size = size; 2993 inode->i_nlink = 0; /* It is unlinked */ 2994#ifndef CONFIG_MMU 2995 error = ramfs_nommu_expand_for_mapping(inode, size); 2996 if (error) 2997 goto put_dentry; 2998#endif 2999 3000 error = -ENFILE; 3001 file = alloc_file(&path, FMODE_WRITE | FMODE_READ, 3002 &shmem_file_operations); 3003 if (!file) 3004 goto put_dentry; 3005 3006 return file; 3007 3008put_dentry: 3009 path_put(&path); 3010put_memory: 3011 shmem_unacct_size(flags, size); 3012 return ERR_PTR(error); 3013} 3014EXPORT_SYMBOL_GPL(shmem_file_setup); 3015 3016/** 3017 * shmem_zero_setup - setup a shared anonymous mapping 3018 * @vma: the vma to be mmapped is prepared by do_mmap_pgoff 3019 */ 3020int shmem_zero_setup(struct vm_area_struct *vma) 3021{ 3022 struct file *file; 3023 loff_t size = vma->vm_end - vma->vm_start; 3024 3025 file = shmem_file_setup("dev/zero", size, vma->vm_flags); 3026 if (IS_ERR(file)) 3027 return PTR_ERR(file); 3028 3029 if (vma->vm_file) 3030 fput(vma->vm_file); 3031 vma->vm_file = file; 3032 vma->vm_ops = &shmem_vm_ops; 3033 vma->vm_flags |= VM_CAN_NONLINEAR; 3034 return 0; 3035} 3036 3037/** 3038 * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags. 3039 * @mapping: the page's address_space 3040 * @index: the page index 3041 * @gfp: the page allocator flags to use if allocating 3042 * 3043 * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)", 3044 * with any new page allocations done using the specified allocation flags. 3045 * But read_cache_page_gfp() uses the ->readpage() method: which does not 3046 * suit tmpfs, since it may have pages in swapcache, and needs to find those 3047 * for itself; although drivers/gpu/drm i915 and ttm rely upon this support. 3048 * 3049 * Provide a stub for those callers to start using now, then later 3050 * flesh it out to call shmem_getpage() with additional gfp mask, when 3051 * shmem_file_splice_read() is added and shmem_readpage() is removed. 3052 */ 3053struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, 3054 pgoff_t index, gfp_t gfp) 3055{ 3056 return read_cache_page_gfp(mapping, index, gfp); 3057} 3058EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp); 3059