swapfile.c revision b73d7fcecd93dc15eaa3c45c8c587b613f6673c4
1/* 2 * linux/mm/swapfile.c 3 * 4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 5 * Swap reorganised 29.12.95, Stephen Tweedie 6 */ 7 8#include <linux/mm.h> 9#include <linux/hugetlb.h> 10#include <linux/mman.h> 11#include <linux/slab.h> 12#include <linux/kernel_stat.h> 13#include <linux/swap.h> 14#include <linux/vmalloc.h> 15#include <linux/pagemap.h> 16#include <linux/namei.h> 17#include <linux/shm.h> 18#include <linux/blkdev.h> 19#include <linux/random.h> 20#include <linux/writeback.h> 21#include <linux/proc_fs.h> 22#include <linux/seq_file.h> 23#include <linux/init.h> 24#include <linux/module.h> 25#include <linux/ksm.h> 26#include <linux/rmap.h> 27#include <linux/security.h> 28#include <linux/backing-dev.h> 29#include <linux/mutex.h> 30#include <linux/capability.h> 31#include <linux/syscalls.h> 32#include <linux/memcontrol.h> 33 34#include <asm/pgtable.h> 35#include <asm/tlbflush.h> 36#include <linux/swapops.h> 37#include <linux/page_cgroup.h> 38 39static bool swap_count_continued(struct swap_info_struct *, pgoff_t, 40 unsigned char); 41static void free_swap_count_continuations(struct swap_info_struct *); 42static sector_t map_swap_entry(swp_entry_t, struct block_device**); 43 44static DEFINE_SPINLOCK(swap_lock); 45static unsigned int nr_swapfiles; 46long nr_swap_pages; 47long total_swap_pages; 48static int least_priority; 49 50static const char Bad_file[] = "Bad swap file entry "; 51static const char Unused_file[] = "Unused swap file entry "; 52static const char Bad_offset[] = "Bad swap offset entry "; 53static const char Unused_offset[] = "Unused swap offset entry "; 54 55static struct swap_list_t swap_list = {-1, -1}; 56 57static struct swap_info_struct *swap_info[MAX_SWAPFILES]; 58 59static DEFINE_MUTEX(swapon_mutex); 60 61static inline unsigned char swap_count(unsigned char ent) 62{ 63 return ent & ~SWAP_HAS_CACHE; /* may include SWAP_HAS_CONT flag */ 64} 65 66/* returns 1 if swap entry is freed */ 67static int 68__try_to_reclaim_swap(struct swap_info_struct *si, unsigned long offset) 69{ 70 swp_entry_t entry = swp_entry(si->type, offset); 71 struct page *page; 72 int ret = 0; 73 74 page = find_get_page(&swapper_space, entry.val); 75 if (!page) 76 return 0; 77 /* 78 * This function is called from scan_swap_map() and it's called 79 * by vmscan.c at reclaiming pages. So, we hold a lock on a page, here. 80 * We have to use trylock for avoiding deadlock. This is a special 81 * case and you should use try_to_free_swap() with explicit lock_page() 82 * in usual operations. 83 */ 84 if (trylock_page(page)) { 85 ret = try_to_free_swap(page); 86 unlock_page(page); 87 } 88 page_cache_release(page); 89 return ret; 90} 91 92/* 93 * We need this because the bdev->unplug_fn can sleep and we cannot 94 * hold swap_lock while calling the unplug_fn. And swap_lock 95 * cannot be turned into a mutex. 96 */ 97static DECLARE_RWSEM(swap_unplug_sem); 98 99void swap_unplug_io_fn(struct backing_dev_info *unused_bdi, struct page *page) 100{ 101 swp_entry_t entry; 102 103 down_read(&swap_unplug_sem); 104 entry.val = page_private(page); 105 if (PageSwapCache(page)) { 106 struct block_device *bdev = swap_info[swp_type(entry)]->bdev; 107 struct backing_dev_info *bdi; 108 109 /* 110 * If the page is removed from swapcache from under us (with a 111 * racy try_to_unuse/swapoff) we need an additional reference 112 * count to avoid reading garbage from page_private(page) above. 113 * If the WARN_ON triggers during a swapoff it maybe the race 114 * condition and it's harmless. However if it triggers without 115 * swapoff it signals a problem. 116 */ 117 WARN_ON(page_count(page) <= 1); 118 119 bdi = bdev->bd_inode->i_mapping->backing_dev_info; 120 blk_run_backing_dev(bdi, page); 121 } 122 up_read(&swap_unplug_sem); 123} 124 125/* 126 * swapon tell device that all the old swap contents can be discarded, 127 * to allow the swap device to optimize its wear-levelling. 128 */ 129static int discard_swap(struct swap_info_struct *si) 130{ 131 struct swap_extent *se; 132 sector_t start_block; 133 sector_t nr_blocks; 134 int err = 0; 135 136 /* Do not discard the swap header page! */ 137 se = &si->first_swap_extent; 138 start_block = (se->start_block + 1) << (PAGE_SHIFT - 9); 139 nr_blocks = ((sector_t)se->nr_pages - 1) << (PAGE_SHIFT - 9); 140 if (nr_blocks) { 141 err = blkdev_issue_discard(si->bdev, start_block, 142 nr_blocks, GFP_KERNEL, 143 BLKDEV_IFL_WAIT | BLKDEV_IFL_BARRIER); 144 if (err) 145 return err; 146 cond_resched(); 147 } 148 149 list_for_each_entry(se, &si->first_swap_extent.list, list) { 150 start_block = se->start_block << (PAGE_SHIFT - 9); 151 nr_blocks = (sector_t)se->nr_pages << (PAGE_SHIFT - 9); 152 153 err = blkdev_issue_discard(si->bdev, start_block, 154 nr_blocks, GFP_KERNEL, 155 BLKDEV_IFL_WAIT | BLKDEV_IFL_BARRIER); 156 if (err) 157 break; 158 159 cond_resched(); 160 } 161 return err; /* That will often be -EOPNOTSUPP */ 162} 163 164/* 165 * swap allocation tell device that a cluster of swap can now be discarded, 166 * to allow the swap device to optimize its wear-levelling. 167 */ 168static void discard_swap_cluster(struct swap_info_struct *si, 169 pgoff_t start_page, pgoff_t nr_pages) 170{ 171 struct swap_extent *se = si->curr_swap_extent; 172 int found_extent = 0; 173 174 while (nr_pages) { 175 struct list_head *lh; 176 177 if (se->start_page <= start_page && 178 start_page < se->start_page + se->nr_pages) { 179 pgoff_t offset = start_page - se->start_page; 180 sector_t start_block = se->start_block + offset; 181 sector_t nr_blocks = se->nr_pages - offset; 182 183 if (nr_blocks > nr_pages) 184 nr_blocks = nr_pages; 185 start_page += nr_blocks; 186 nr_pages -= nr_blocks; 187 188 if (!found_extent++) 189 si->curr_swap_extent = se; 190 191 start_block <<= PAGE_SHIFT - 9; 192 nr_blocks <<= PAGE_SHIFT - 9; 193 if (blkdev_issue_discard(si->bdev, start_block, 194 nr_blocks, GFP_NOIO, BLKDEV_IFL_WAIT | 195 BLKDEV_IFL_BARRIER)) 196 break; 197 } 198 199 lh = se->list.next; 200 se = list_entry(lh, struct swap_extent, list); 201 } 202} 203 204static int wait_for_discard(void *word) 205{ 206 schedule(); 207 return 0; 208} 209 210#define SWAPFILE_CLUSTER 256 211#define LATENCY_LIMIT 256 212 213static inline unsigned long scan_swap_map(struct swap_info_struct *si, 214 unsigned char usage) 215{ 216 unsigned long offset; 217 unsigned long scan_base; 218 unsigned long last_in_cluster = 0; 219 int latency_ration = LATENCY_LIMIT; 220 int found_free_cluster = 0; 221 222 /* 223 * We try to cluster swap pages by allocating them sequentially 224 * in swap. Once we've allocated SWAPFILE_CLUSTER pages this 225 * way, however, we resort to first-free allocation, starting 226 * a new cluster. This prevents us from scattering swap pages 227 * all over the entire swap partition, so that we reduce 228 * overall disk seek times between swap pages. -- sct 229 * But we do now try to find an empty cluster. -Andrea 230 * And we let swap pages go all over an SSD partition. Hugh 231 */ 232 233 si->flags += SWP_SCANNING; 234 scan_base = offset = si->cluster_next; 235 236 if (unlikely(!si->cluster_nr--)) { 237 if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) { 238 si->cluster_nr = SWAPFILE_CLUSTER - 1; 239 goto checks; 240 } 241 if (si->flags & SWP_DISCARDABLE) { 242 /* 243 * Start range check on racing allocations, in case 244 * they overlap the cluster we eventually decide on 245 * (we scan without swap_lock to allow preemption). 246 * It's hardly conceivable that cluster_nr could be 247 * wrapped during our scan, but don't depend on it. 248 */ 249 if (si->lowest_alloc) 250 goto checks; 251 si->lowest_alloc = si->max; 252 si->highest_alloc = 0; 253 } 254 spin_unlock(&swap_lock); 255 256 /* 257 * If seek is expensive, start searching for new cluster from 258 * start of partition, to minimize the span of allocated swap. 259 * But if seek is cheap, search from our current position, so 260 * that swap is allocated from all over the partition: if the 261 * Flash Translation Layer only remaps within limited zones, 262 * we don't want to wear out the first zone too quickly. 263 */ 264 if (!(si->flags & SWP_SOLIDSTATE)) 265 scan_base = offset = si->lowest_bit; 266 last_in_cluster = offset + SWAPFILE_CLUSTER - 1; 267 268 /* Locate the first empty (unaligned) cluster */ 269 for (; last_in_cluster <= si->highest_bit; offset++) { 270 if (si->swap_map[offset]) 271 last_in_cluster = offset + SWAPFILE_CLUSTER; 272 else if (offset == last_in_cluster) { 273 spin_lock(&swap_lock); 274 offset -= SWAPFILE_CLUSTER - 1; 275 si->cluster_next = offset; 276 si->cluster_nr = SWAPFILE_CLUSTER - 1; 277 found_free_cluster = 1; 278 goto checks; 279 } 280 if (unlikely(--latency_ration < 0)) { 281 cond_resched(); 282 latency_ration = LATENCY_LIMIT; 283 } 284 } 285 286 offset = si->lowest_bit; 287 last_in_cluster = offset + SWAPFILE_CLUSTER - 1; 288 289 /* Locate the first empty (unaligned) cluster */ 290 for (; last_in_cluster < scan_base; offset++) { 291 if (si->swap_map[offset]) 292 last_in_cluster = offset + SWAPFILE_CLUSTER; 293 else if (offset == last_in_cluster) { 294 spin_lock(&swap_lock); 295 offset -= SWAPFILE_CLUSTER - 1; 296 si->cluster_next = offset; 297 si->cluster_nr = SWAPFILE_CLUSTER - 1; 298 found_free_cluster = 1; 299 goto checks; 300 } 301 if (unlikely(--latency_ration < 0)) { 302 cond_resched(); 303 latency_ration = LATENCY_LIMIT; 304 } 305 } 306 307 offset = scan_base; 308 spin_lock(&swap_lock); 309 si->cluster_nr = SWAPFILE_CLUSTER - 1; 310 si->lowest_alloc = 0; 311 } 312 313checks: 314 if (!(si->flags & SWP_WRITEOK)) 315 goto no_page; 316 if (!si->highest_bit) 317 goto no_page; 318 if (offset > si->highest_bit) 319 scan_base = offset = si->lowest_bit; 320 321 /* reuse swap entry of cache-only swap if not busy. */ 322 if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) { 323 int swap_was_freed; 324 spin_unlock(&swap_lock); 325 swap_was_freed = __try_to_reclaim_swap(si, offset); 326 spin_lock(&swap_lock); 327 /* entry was freed successfully, try to use this again */ 328 if (swap_was_freed) 329 goto checks; 330 goto scan; /* check next one */ 331 } 332 333 if (si->swap_map[offset]) 334 goto scan; 335 336 if (offset == si->lowest_bit) 337 si->lowest_bit++; 338 if (offset == si->highest_bit) 339 si->highest_bit--; 340 si->inuse_pages++; 341 if (si->inuse_pages == si->pages) { 342 si->lowest_bit = si->max; 343 si->highest_bit = 0; 344 } 345 si->swap_map[offset] = usage; 346 si->cluster_next = offset + 1; 347 si->flags -= SWP_SCANNING; 348 349 if (si->lowest_alloc) { 350 /* 351 * Only set when SWP_DISCARDABLE, and there's a scan 352 * for a free cluster in progress or just completed. 353 */ 354 if (found_free_cluster) { 355 /* 356 * To optimize wear-levelling, discard the 357 * old data of the cluster, taking care not to 358 * discard any of its pages that have already 359 * been allocated by racing tasks (offset has 360 * already stepped over any at the beginning). 361 */ 362 if (offset < si->highest_alloc && 363 si->lowest_alloc <= last_in_cluster) 364 last_in_cluster = si->lowest_alloc - 1; 365 si->flags |= SWP_DISCARDING; 366 spin_unlock(&swap_lock); 367 368 if (offset < last_in_cluster) 369 discard_swap_cluster(si, offset, 370 last_in_cluster - offset + 1); 371 372 spin_lock(&swap_lock); 373 si->lowest_alloc = 0; 374 si->flags &= ~SWP_DISCARDING; 375 376 smp_mb(); /* wake_up_bit advises this */ 377 wake_up_bit(&si->flags, ilog2(SWP_DISCARDING)); 378 379 } else if (si->flags & SWP_DISCARDING) { 380 /* 381 * Delay using pages allocated by racing tasks 382 * until the whole discard has been issued. We 383 * could defer that delay until swap_writepage, 384 * but it's easier to keep this self-contained. 385 */ 386 spin_unlock(&swap_lock); 387 wait_on_bit(&si->flags, ilog2(SWP_DISCARDING), 388 wait_for_discard, TASK_UNINTERRUPTIBLE); 389 spin_lock(&swap_lock); 390 } else { 391 /* 392 * Note pages allocated by racing tasks while 393 * scan for a free cluster is in progress, so 394 * that its final discard can exclude them. 395 */ 396 if (offset < si->lowest_alloc) 397 si->lowest_alloc = offset; 398 if (offset > si->highest_alloc) 399 si->highest_alloc = offset; 400 } 401 } 402 return offset; 403 404scan: 405 spin_unlock(&swap_lock); 406 while (++offset <= si->highest_bit) { 407 if (!si->swap_map[offset]) { 408 spin_lock(&swap_lock); 409 goto checks; 410 } 411 if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) { 412 spin_lock(&swap_lock); 413 goto checks; 414 } 415 if (unlikely(--latency_ration < 0)) { 416 cond_resched(); 417 latency_ration = LATENCY_LIMIT; 418 } 419 } 420 offset = si->lowest_bit; 421 while (++offset < scan_base) { 422 if (!si->swap_map[offset]) { 423 spin_lock(&swap_lock); 424 goto checks; 425 } 426 if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) { 427 spin_lock(&swap_lock); 428 goto checks; 429 } 430 if (unlikely(--latency_ration < 0)) { 431 cond_resched(); 432 latency_ration = LATENCY_LIMIT; 433 } 434 } 435 spin_lock(&swap_lock); 436 437no_page: 438 si->flags -= SWP_SCANNING; 439 return 0; 440} 441 442swp_entry_t get_swap_page(void) 443{ 444 struct swap_info_struct *si; 445 pgoff_t offset; 446 int type, next; 447 int wrapped = 0; 448 449 spin_lock(&swap_lock); 450 if (nr_swap_pages <= 0) 451 goto noswap; 452 nr_swap_pages--; 453 454 for (type = swap_list.next; type >= 0 && wrapped < 2; type = next) { 455 si = swap_info[type]; 456 next = si->next; 457 if (next < 0 || 458 (!wrapped && si->prio != swap_info[next]->prio)) { 459 next = swap_list.head; 460 wrapped++; 461 } 462 463 if (!si->highest_bit) 464 continue; 465 if (!(si->flags & SWP_WRITEOK)) 466 continue; 467 468 swap_list.next = next; 469 /* This is called for allocating swap entry for cache */ 470 offset = scan_swap_map(si, SWAP_HAS_CACHE); 471 if (offset) { 472 spin_unlock(&swap_lock); 473 return swp_entry(type, offset); 474 } 475 next = swap_list.next; 476 } 477 478 nr_swap_pages++; 479noswap: 480 spin_unlock(&swap_lock); 481 return (swp_entry_t) {0}; 482} 483 484/* The only caller of this function is now susupend routine */ 485swp_entry_t get_swap_page_of_type(int type) 486{ 487 struct swap_info_struct *si; 488 pgoff_t offset; 489 490 spin_lock(&swap_lock); 491 si = swap_info[type]; 492 if (si && (si->flags & SWP_WRITEOK)) { 493 nr_swap_pages--; 494 /* This is called for allocating swap entry, not cache */ 495 offset = scan_swap_map(si, 1); 496 if (offset) { 497 spin_unlock(&swap_lock); 498 return swp_entry(type, offset); 499 } 500 nr_swap_pages++; 501 } 502 spin_unlock(&swap_lock); 503 return (swp_entry_t) {0}; 504} 505 506static struct swap_info_struct *swap_info_get(swp_entry_t entry) 507{ 508 struct swap_info_struct *p; 509 unsigned long offset, type; 510 511 if (!entry.val) 512 goto out; 513 type = swp_type(entry); 514 if (type >= nr_swapfiles) 515 goto bad_nofile; 516 p = swap_info[type]; 517 if (!(p->flags & SWP_USED)) 518 goto bad_device; 519 offset = swp_offset(entry); 520 if (offset >= p->max) 521 goto bad_offset; 522 if (!p->swap_map[offset]) 523 goto bad_free; 524 spin_lock(&swap_lock); 525 return p; 526 527bad_free: 528 printk(KERN_ERR "swap_free: %s%08lx\n", Unused_offset, entry.val); 529 goto out; 530bad_offset: 531 printk(KERN_ERR "swap_free: %s%08lx\n", Bad_offset, entry.val); 532 goto out; 533bad_device: 534 printk(KERN_ERR "swap_free: %s%08lx\n", Unused_file, entry.val); 535 goto out; 536bad_nofile: 537 printk(KERN_ERR "swap_free: %s%08lx\n", Bad_file, entry.val); 538out: 539 return NULL; 540} 541 542static unsigned char swap_entry_free(struct swap_info_struct *p, 543 swp_entry_t entry, unsigned char usage) 544{ 545 unsigned long offset = swp_offset(entry); 546 unsigned char count; 547 unsigned char has_cache; 548 549 count = p->swap_map[offset]; 550 has_cache = count & SWAP_HAS_CACHE; 551 count &= ~SWAP_HAS_CACHE; 552 553 if (usage == SWAP_HAS_CACHE) { 554 VM_BUG_ON(!has_cache); 555 has_cache = 0; 556 } else if (count == SWAP_MAP_SHMEM) { 557 /* 558 * Or we could insist on shmem.c using a special 559 * swap_shmem_free() and free_shmem_swap_and_cache()... 560 */ 561 count = 0; 562 } else if ((count & ~COUNT_CONTINUED) <= SWAP_MAP_MAX) { 563 if (count == COUNT_CONTINUED) { 564 if (swap_count_continued(p, offset, count)) 565 count = SWAP_MAP_MAX | COUNT_CONTINUED; 566 else 567 count = SWAP_MAP_MAX; 568 } else 569 count--; 570 } 571 572 if (!count) 573 mem_cgroup_uncharge_swap(entry); 574 575 usage = count | has_cache; 576 p->swap_map[offset] = usage; 577 578 /* free if no reference */ 579 if (!usage) { 580 struct gendisk *disk = p->bdev->bd_disk; 581 if (offset < p->lowest_bit) 582 p->lowest_bit = offset; 583 if (offset > p->highest_bit) 584 p->highest_bit = offset; 585 if (swap_list.next >= 0 && 586 p->prio > swap_info[swap_list.next]->prio) 587 swap_list.next = p->type; 588 nr_swap_pages++; 589 p->inuse_pages--; 590 if ((p->flags & SWP_BLKDEV) && 591 disk->fops->swap_slot_free_notify) 592 disk->fops->swap_slot_free_notify(p->bdev, offset); 593 } 594 595 return usage; 596} 597 598/* 599 * Caller has made sure that the swapdevice corresponding to entry 600 * is still around or has not been recycled. 601 */ 602void swap_free(swp_entry_t entry) 603{ 604 struct swap_info_struct *p; 605 606 p = swap_info_get(entry); 607 if (p) { 608 swap_entry_free(p, entry, 1); 609 spin_unlock(&swap_lock); 610 } 611} 612 613/* 614 * Called after dropping swapcache to decrease refcnt to swap entries. 615 */ 616void swapcache_free(swp_entry_t entry, struct page *page) 617{ 618 struct swap_info_struct *p; 619 unsigned char count; 620 621 p = swap_info_get(entry); 622 if (p) { 623 count = swap_entry_free(p, entry, SWAP_HAS_CACHE); 624 if (page) 625 mem_cgroup_uncharge_swapcache(page, entry, count != 0); 626 spin_unlock(&swap_lock); 627 } 628} 629 630/* 631 * How many references to page are currently swapped out? 632 * This does not give an exact answer when swap count is continued, 633 * but does include the high COUNT_CONTINUED flag to allow for that. 634 */ 635static inline int page_swapcount(struct page *page) 636{ 637 int count = 0; 638 struct swap_info_struct *p; 639 swp_entry_t entry; 640 641 entry.val = page_private(page); 642 p = swap_info_get(entry); 643 if (p) { 644 count = swap_count(p->swap_map[swp_offset(entry)]); 645 spin_unlock(&swap_lock); 646 } 647 return count; 648} 649 650/* 651 * We can write to an anon page without COW if there are no other references 652 * to it. And as a side-effect, free up its swap: because the old content 653 * on disk will never be read, and seeking back there to write new content 654 * later would only waste time away from clustering. 655 */ 656int reuse_swap_page(struct page *page) 657{ 658 int count; 659 660 VM_BUG_ON(!PageLocked(page)); 661 if (unlikely(PageKsm(page))) 662 return 0; 663 count = page_mapcount(page); 664 if (count <= 1 && PageSwapCache(page)) { 665 count += page_swapcount(page); 666 if (count == 1 && !PageWriteback(page)) { 667 delete_from_swap_cache(page); 668 SetPageDirty(page); 669 } 670 } 671 return count <= 1; 672} 673 674/* 675 * If swap is getting full, or if there are no more mappings of this page, 676 * then try_to_free_swap is called to free its swap space. 677 */ 678int try_to_free_swap(struct page *page) 679{ 680 VM_BUG_ON(!PageLocked(page)); 681 682 if (!PageSwapCache(page)) 683 return 0; 684 if (PageWriteback(page)) 685 return 0; 686 if (page_swapcount(page)) 687 return 0; 688 689 /* 690 * Once hibernation has begun to create its image of memory, 691 * there's a danger that one of the calls to try_to_free_swap() 692 * - most probably a call from __try_to_reclaim_swap() while 693 * hibernation is allocating its own swap pages for the image, 694 * but conceivably even a call from memory reclaim - will free 695 * the swap from a page which has already been recorded in the 696 * image as a clean swapcache page, and then reuse its swap for 697 * another page of the image. On waking from hibernation, the 698 * original page might be freed under memory pressure, then 699 * later read back in from swap, now with the wrong data. 700 * 701 * Hibernation clears bits from gfp_allowed_mask to prevent 702 * memory reclaim from writing to disk, so check that here. 703 */ 704 if (!(gfp_allowed_mask & __GFP_IO)) 705 return 0; 706 707 delete_from_swap_cache(page); 708 SetPageDirty(page); 709 return 1; 710} 711 712/* 713 * Free the swap entry like above, but also try to 714 * free the page cache entry if it is the last user. 715 */ 716int free_swap_and_cache(swp_entry_t entry) 717{ 718 struct swap_info_struct *p; 719 struct page *page = NULL; 720 721 if (non_swap_entry(entry)) 722 return 1; 723 724 p = swap_info_get(entry); 725 if (p) { 726 if (swap_entry_free(p, entry, 1) == SWAP_HAS_CACHE) { 727 page = find_get_page(&swapper_space, entry.val); 728 if (page && !trylock_page(page)) { 729 page_cache_release(page); 730 page = NULL; 731 } 732 } 733 spin_unlock(&swap_lock); 734 } 735 if (page) { 736 /* 737 * Not mapped elsewhere, or swap space full? Free it! 738 * Also recheck PageSwapCache now page is locked (above). 739 */ 740 if (PageSwapCache(page) && !PageWriteback(page) && 741 (!page_mapped(page) || vm_swap_full())) { 742 delete_from_swap_cache(page); 743 SetPageDirty(page); 744 } 745 unlock_page(page); 746 page_cache_release(page); 747 } 748 return p != NULL; 749} 750 751#ifdef CONFIG_CGROUP_MEM_RES_CTLR 752/** 753 * mem_cgroup_count_swap_user - count the user of a swap entry 754 * @ent: the swap entry to be checked 755 * @pagep: the pointer for the swap cache page of the entry to be stored 756 * 757 * Returns the number of the user of the swap entry. The number is valid only 758 * for swaps of anonymous pages. 759 * If the entry is found on swap cache, the page is stored to pagep with 760 * refcount of it being incremented. 761 */ 762int mem_cgroup_count_swap_user(swp_entry_t ent, struct page **pagep) 763{ 764 struct page *page; 765 struct swap_info_struct *p; 766 int count = 0; 767 768 page = find_get_page(&swapper_space, ent.val); 769 if (page) 770 count += page_mapcount(page); 771 p = swap_info_get(ent); 772 if (p) { 773 count += swap_count(p->swap_map[swp_offset(ent)]); 774 spin_unlock(&swap_lock); 775 } 776 777 *pagep = page; 778 return count; 779} 780#endif 781 782#ifdef CONFIG_HIBERNATION 783/* 784 * Find the swap type that corresponds to given device (if any). 785 * 786 * @offset - number of the PAGE_SIZE-sized block of the device, starting 787 * from 0, in which the swap header is expected to be located. 788 * 789 * This is needed for the suspend to disk (aka swsusp). 790 */ 791int swap_type_of(dev_t device, sector_t offset, struct block_device **bdev_p) 792{ 793 struct block_device *bdev = NULL; 794 int type; 795 796 if (device) 797 bdev = bdget(device); 798 799 spin_lock(&swap_lock); 800 for (type = 0; type < nr_swapfiles; type++) { 801 struct swap_info_struct *sis = swap_info[type]; 802 803 if (!(sis->flags & SWP_WRITEOK)) 804 continue; 805 806 if (!bdev) { 807 if (bdev_p) 808 *bdev_p = bdgrab(sis->bdev); 809 810 spin_unlock(&swap_lock); 811 return type; 812 } 813 if (bdev == sis->bdev) { 814 struct swap_extent *se = &sis->first_swap_extent; 815 816 if (se->start_block == offset) { 817 if (bdev_p) 818 *bdev_p = bdgrab(sis->bdev); 819 820 spin_unlock(&swap_lock); 821 bdput(bdev); 822 return type; 823 } 824 } 825 } 826 spin_unlock(&swap_lock); 827 if (bdev) 828 bdput(bdev); 829 830 return -ENODEV; 831} 832 833/* 834 * Get the (PAGE_SIZE) block corresponding to given offset on the swapdev 835 * corresponding to given index in swap_info (swap type). 836 */ 837sector_t swapdev_block(int type, pgoff_t offset) 838{ 839 struct block_device *bdev; 840 841 if ((unsigned int)type >= nr_swapfiles) 842 return 0; 843 if (!(swap_info[type]->flags & SWP_WRITEOK)) 844 return 0; 845 return map_swap_entry(swp_entry(type, offset), &bdev); 846} 847 848/* 849 * Return either the total number of swap pages of given type, or the number 850 * of free pages of that type (depending on @free) 851 * 852 * This is needed for software suspend 853 */ 854unsigned int count_swap_pages(int type, int free) 855{ 856 unsigned int n = 0; 857 858 spin_lock(&swap_lock); 859 if ((unsigned int)type < nr_swapfiles) { 860 struct swap_info_struct *sis = swap_info[type]; 861 862 if (sis->flags & SWP_WRITEOK) { 863 n = sis->pages; 864 if (free) 865 n -= sis->inuse_pages; 866 } 867 } 868 spin_unlock(&swap_lock); 869 return n; 870} 871#endif /* CONFIG_HIBERNATION */ 872 873/* 874 * No need to decide whether this PTE shares the swap entry with others, 875 * just let do_wp_page work it out if a write is requested later - to 876 * force COW, vm_page_prot omits write permission from any private vma. 877 */ 878static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, 879 unsigned long addr, swp_entry_t entry, struct page *page) 880{ 881 struct mem_cgroup *ptr = NULL; 882 spinlock_t *ptl; 883 pte_t *pte; 884 int ret = 1; 885 886 if (mem_cgroup_try_charge_swapin(vma->vm_mm, page, GFP_KERNEL, &ptr)) { 887 ret = -ENOMEM; 888 goto out_nolock; 889 } 890 891 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 892 if (unlikely(!pte_same(*pte, swp_entry_to_pte(entry)))) { 893 if (ret > 0) 894 mem_cgroup_cancel_charge_swapin(ptr); 895 ret = 0; 896 goto out; 897 } 898 899 dec_mm_counter(vma->vm_mm, MM_SWAPENTS); 900 inc_mm_counter(vma->vm_mm, MM_ANONPAGES); 901 get_page(page); 902 set_pte_at(vma->vm_mm, addr, pte, 903 pte_mkold(mk_pte(page, vma->vm_page_prot))); 904 page_add_anon_rmap(page, vma, addr); 905 mem_cgroup_commit_charge_swapin(page, ptr); 906 swap_free(entry); 907 /* 908 * Move the page to the active list so it is not 909 * immediately swapped out again after swapon. 910 */ 911 activate_page(page); 912out: 913 pte_unmap_unlock(pte, ptl); 914out_nolock: 915 return ret; 916} 917 918static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, 919 unsigned long addr, unsigned long end, 920 swp_entry_t entry, struct page *page) 921{ 922 pte_t swp_pte = swp_entry_to_pte(entry); 923 pte_t *pte; 924 int ret = 0; 925 926 /* 927 * We don't actually need pte lock while scanning for swp_pte: since 928 * we hold page lock and mmap_sem, swp_pte cannot be inserted into the 929 * page table while we're scanning; though it could get zapped, and on 930 * some architectures (e.g. x86_32 with PAE) we might catch a glimpse 931 * of unmatched parts which look like swp_pte, so unuse_pte must 932 * recheck under pte lock. Scanning without pte lock lets it be 933 * preemptible whenever CONFIG_PREEMPT but not CONFIG_HIGHPTE. 934 */ 935 pte = pte_offset_map(pmd, addr); 936 do { 937 /* 938 * swapoff spends a _lot_ of time in this loop! 939 * Test inline before going to call unuse_pte. 940 */ 941 if (unlikely(pte_same(*pte, swp_pte))) { 942 pte_unmap(pte); 943 ret = unuse_pte(vma, pmd, addr, entry, page); 944 if (ret) 945 goto out; 946 pte = pte_offset_map(pmd, addr); 947 } 948 } while (pte++, addr += PAGE_SIZE, addr != end); 949 pte_unmap(pte - 1); 950out: 951 return ret; 952} 953 954static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud, 955 unsigned long addr, unsigned long end, 956 swp_entry_t entry, struct page *page) 957{ 958 pmd_t *pmd; 959 unsigned long next; 960 int ret; 961 962 pmd = pmd_offset(pud, addr); 963 do { 964 next = pmd_addr_end(addr, end); 965 if (pmd_none_or_clear_bad(pmd)) 966 continue; 967 ret = unuse_pte_range(vma, pmd, addr, next, entry, page); 968 if (ret) 969 return ret; 970 } while (pmd++, addr = next, addr != end); 971 return 0; 972} 973 974static inline int unuse_pud_range(struct vm_area_struct *vma, pgd_t *pgd, 975 unsigned long addr, unsigned long end, 976 swp_entry_t entry, struct page *page) 977{ 978 pud_t *pud; 979 unsigned long next; 980 int ret; 981 982 pud = pud_offset(pgd, addr); 983 do { 984 next = pud_addr_end(addr, end); 985 if (pud_none_or_clear_bad(pud)) 986 continue; 987 ret = unuse_pmd_range(vma, pud, addr, next, entry, page); 988 if (ret) 989 return ret; 990 } while (pud++, addr = next, addr != end); 991 return 0; 992} 993 994static int unuse_vma(struct vm_area_struct *vma, 995 swp_entry_t entry, struct page *page) 996{ 997 pgd_t *pgd; 998 unsigned long addr, end, next; 999 int ret; 1000 1001 if (page_anon_vma(page)) { 1002 addr = page_address_in_vma(page, vma); 1003 if (addr == -EFAULT) 1004 return 0; 1005 else 1006 end = addr + PAGE_SIZE; 1007 } else { 1008 addr = vma->vm_start; 1009 end = vma->vm_end; 1010 } 1011 1012 pgd = pgd_offset(vma->vm_mm, addr); 1013 do { 1014 next = pgd_addr_end(addr, end); 1015 if (pgd_none_or_clear_bad(pgd)) 1016 continue; 1017 ret = unuse_pud_range(vma, pgd, addr, next, entry, page); 1018 if (ret) 1019 return ret; 1020 } while (pgd++, addr = next, addr != end); 1021 return 0; 1022} 1023 1024static int unuse_mm(struct mm_struct *mm, 1025 swp_entry_t entry, struct page *page) 1026{ 1027 struct vm_area_struct *vma; 1028 int ret = 0; 1029 1030 if (!down_read_trylock(&mm->mmap_sem)) { 1031 /* 1032 * Activate page so shrink_inactive_list is unlikely to unmap 1033 * its ptes while lock is dropped, so swapoff can make progress. 1034 */ 1035 activate_page(page); 1036 unlock_page(page); 1037 down_read(&mm->mmap_sem); 1038 lock_page(page); 1039 } 1040 for (vma = mm->mmap; vma; vma = vma->vm_next) { 1041 if (vma->anon_vma && (ret = unuse_vma(vma, entry, page))) 1042 break; 1043 } 1044 up_read(&mm->mmap_sem); 1045 return (ret < 0)? ret: 0; 1046} 1047 1048/* 1049 * Scan swap_map from current position to next entry still in use. 1050 * Recycle to start on reaching the end, returning 0 when empty. 1051 */ 1052static unsigned int find_next_to_unuse(struct swap_info_struct *si, 1053 unsigned int prev) 1054{ 1055 unsigned int max = si->max; 1056 unsigned int i = prev; 1057 unsigned char count; 1058 1059 /* 1060 * No need for swap_lock here: we're just looking 1061 * for whether an entry is in use, not modifying it; false 1062 * hits are okay, and sys_swapoff() has already prevented new 1063 * allocations from this area (while holding swap_lock). 1064 */ 1065 for (;;) { 1066 if (++i >= max) { 1067 if (!prev) { 1068 i = 0; 1069 break; 1070 } 1071 /* 1072 * No entries in use at top of swap_map, 1073 * loop back to start and recheck there. 1074 */ 1075 max = prev + 1; 1076 prev = 0; 1077 i = 1; 1078 } 1079 count = si->swap_map[i]; 1080 if (count && swap_count(count) != SWAP_MAP_BAD) 1081 break; 1082 } 1083 return i; 1084} 1085 1086/* 1087 * We completely avoid races by reading each swap page in advance, 1088 * and then search for the process using it. All the necessary 1089 * page table adjustments can then be made atomically. 1090 */ 1091static int try_to_unuse(unsigned int type) 1092{ 1093 struct swap_info_struct *si = swap_info[type]; 1094 struct mm_struct *start_mm; 1095 unsigned char *swap_map; 1096 unsigned char swcount; 1097 struct page *page; 1098 swp_entry_t entry; 1099 unsigned int i = 0; 1100 int retval = 0; 1101 1102 /* 1103 * When searching mms for an entry, a good strategy is to 1104 * start at the first mm we freed the previous entry from 1105 * (though actually we don't notice whether we or coincidence 1106 * freed the entry). Initialize this start_mm with a hold. 1107 * 1108 * A simpler strategy would be to start at the last mm we 1109 * freed the previous entry from; but that would take less 1110 * advantage of mmlist ordering, which clusters forked mms 1111 * together, child after parent. If we race with dup_mmap(), we 1112 * prefer to resolve parent before child, lest we miss entries 1113 * duplicated after we scanned child: using last mm would invert 1114 * that. 1115 */ 1116 start_mm = &init_mm; 1117 atomic_inc(&init_mm.mm_users); 1118 1119 /* 1120 * Keep on scanning until all entries have gone. Usually, 1121 * one pass through swap_map is enough, but not necessarily: 1122 * there are races when an instance of an entry might be missed. 1123 */ 1124 while ((i = find_next_to_unuse(si, i)) != 0) { 1125 if (signal_pending(current)) { 1126 retval = -EINTR; 1127 break; 1128 } 1129 1130 /* 1131 * Get a page for the entry, using the existing swap 1132 * cache page if there is one. Otherwise, get a clean 1133 * page and read the swap into it. 1134 */ 1135 swap_map = &si->swap_map[i]; 1136 entry = swp_entry(type, i); 1137 page = read_swap_cache_async(entry, 1138 GFP_HIGHUSER_MOVABLE, NULL, 0); 1139 if (!page) { 1140 /* 1141 * Either swap_duplicate() failed because entry 1142 * has been freed independently, and will not be 1143 * reused since sys_swapoff() already disabled 1144 * allocation from here, or alloc_page() failed. 1145 */ 1146 if (!*swap_map) 1147 continue; 1148 retval = -ENOMEM; 1149 break; 1150 } 1151 1152 /* 1153 * Don't hold on to start_mm if it looks like exiting. 1154 */ 1155 if (atomic_read(&start_mm->mm_users) == 1) { 1156 mmput(start_mm); 1157 start_mm = &init_mm; 1158 atomic_inc(&init_mm.mm_users); 1159 } 1160 1161 /* 1162 * Wait for and lock page. When do_swap_page races with 1163 * try_to_unuse, do_swap_page can handle the fault much 1164 * faster than try_to_unuse can locate the entry. This 1165 * apparently redundant "wait_on_page_locked" lets try_to_unuse 1166 * defer to do_swap_page in such a case - in some tests, 1167 * do_swap_page and try_to_unuse repeatedly compete. 1168 */ 1169 wait_on_page_locked(page); 1170 wait_on_page_writeback(page); 1171 lock_page(page); 1172 wait_on_page_writeback(page); 1173 1174 /* 1175 * Remove all references to entry. 1176 */ 1177 swcount = *swap_map; 1178 if (swap_count(swcount) == SWAP_MAP_SHMEM) { 1179 retval = shmem_unuse(entry, page); 1180 /* page has already been unlocked and released */ 1181 if (retval < 0) 1182 break; 1183 continue; 1184 } 1185 if (swap_count(swcount) && start_mm != &init_mm) 1186 retval = unuse_mm(start_mm, entry, page); 1187 1188 if (swap_count(*swap_map)) { 1189 int set_start_mm = (*swap_map >= swcount); 1190 struct list_head *p = &start_mm->mmlist; 1191 struct mm_struct *new_start_mm = start_mm; 1192 struct mm_struct *prev_mm = start_mm; 1193 struct mm_struct *mm; 1194 1195 atomic_inc(&new_start_mm->mm_users); 1196 atomic_inc(&prev_mm->mm_users); 1197 spin_lock(&mmlist_lock); 1198 while (swap_count(*swap_map) && !retval && 1199 (p = p->next) != &start_mm->mmlist) { 1200 mm = list_entry(p, struct mm_struct, mmlist); 1201 if (!atomic_inc_not_zero(&mm->mm_users)) 1202 continue; 1203 spin_unlock(&mmlist_lock); 1204 mmput(prev_mm); 1205 prev_mm = mm; 1206 1207 cond_resched(); 1208 1209 swcount = *swap_map; 1210 if (!swap_count(swcount)) /* any usage ? */ 1211 ; 1212 else if (mm == &init_mm) 1213 set_start_mm = 1; 1214 else 1215 retval = unuse_mm(mm, entry, page); 1216 1217 if (set_start_mm && *swap_map < swcount) { 1218 mmput(new_start_mm); 1219 atomic_inc(&mm->mm_users); 1220 new_start_mm = mm; 1221 set_start_mm = 0; 1222 } 1223 spin_lock(&mmlist_lock); 1224 } 1225 spin_unlock(&mmlist_lock); 1226 mmput(prev_mm); 1227 mmput(start_mm); 1228 start_mm = new_start_mm; 1229 } 1230 if (retval) { 1231 unlock_page(page); 1232 page_cache_release(page); 1233 break; 1234 } 1235 1236 /* 1237 * If a reference remains (rare), we would like to leave 1238 * the page in the swap cache; but try_to_unmap could 1239 * then re-duplicate the entry once we drop page lock, 1240 * so we might loop indefinitely; also, that page could 1241 * not be swapped out to other storage meanwhile. So: 1242 * delete from cache even if there's another reference, 1243 * after ensuring that the data has been saved to disk - 1244 * since if the reference remains (rarer), it will be 1245 * read from disk into another page. Splitting into two 1246 * pages would be incorrect if swap supported "shared 1247 * private" pages, but they are handled by tmpfs files. 1248 * 1249 * Given how unuse_vma() targets one particular offset 1250 * in an anon_vma, once the anon_vma has been determined, 1251 * this splitting happens to be just what is needed to 1252 * handle where KSM pages have been swapped out: re-reading 1253 * is unnecessarily slow, but we can fix that later on. 1254 */ 1255 if (swap_count(*swap_map) && 1256 PageDirty(page) && PageSwapCache(page)) { 1257 struct writeback_control wbc = { 1258 .sync_mode = WB_SYNC_NONE, 1259 }; 1260 1261 swap_writepage(page, &wbc); 1262 lock_page(page); 1263 wait_on_page_writeback(page); 1264 } 1265 1266 /* 1267 * It is conceivable that a racing task removed this page from 1268 * swap cache just before we acquired the page lock at the top, 1269 * or while we dropped it in unuse_mm(). The page might even 1270 * be back in swap cache on another swap area: that we must not 1271 * delete, since it may not have been written out to swap yet. 1272 */ 1273 if (PageSwapCache(page) && 1274 likely(page_private(page) == entry.val)) 1275 delete_from_swap_cache(page); 1276 1277 /* 1278 * So we could skip searching mms once swap count went 1279 * to 1, we did not mark any present ptes as dirty: must 1280 * mark page dirty so shrink_page_list will preserve it. 1281 */ 1282 SetPageDirty(page); 1283 unlock_page(page); 1284 page_cache_release(page); 1285 1286 /* 1287 * Make sure that we aren't completely killing 1288 * interactive performance. 1289 */ 1290 cond_resched(); 1291 } 1292 1293 mmput(start_mm); 1294 return retval; 1295} 1296 1297/* 1298 * After a successful try_to_unuse, if no swap is now in use, we know 1299 * we can empty the mmlist. swap_lock must be held on entry and exit. 1300 * Note that mmlist_lock nests inside swap_lock, and an mm must be 1301 * added to the mmlist just after page_duplicate - before would be racy. 1302 */ 1303static void drain_mmlist(void) 1304{ 1305 struct list_head *p, *next; 1306 unsigned int type; 1307 1308 for (type = 0; type < nr_swapfiles; type++) 1309 if (swap_info[type]->inuse_pages) 1310 return; 1311 spin_lock(&mmlist_lock); 1312 list_for_each_safe(p, next, &init_mm.mmlist) 1313 list_del_init(p); 1314 spin_unlock(&mmlist_lock); 1315} 1316 1317/* 1318 * Use this swapdev's extent info to locate the (PAGE_SIZE) block which 1319 * corresponds to page offset for the specified swap entry. 1320 * Note that the type of this function is sector_t, but it returns page offset 1321 * into the bdev, not sector offset. 1322 */ 1323static sector_t map_swap_entry(swp_entry_t entry, struct block_device **bdev) 1324{ 1325 struct swap_info_struct *sis; 1326 struct swap_extent *start_se; 1327 struct swap_extent *se; 1328 pgoff_t offset; 1329 1330 sis = swap_info[swp_type(entry)]; 1331 *bdev = sis->bdev; 1332 1333 offset = swp_offset(entry); 1334 start_se = sis->curr_swap_extent; 1335 se = start_se; 1336 1337 for ( ; ; ) { 1338 struct list_head *lh; 1339 1340 if (se->start_page <= offset && 1341 offset < (se->start_page + se->nr_pages)) { 1342 return se->start_block + (offset - se->start_page); 1343 } 1344 lh = se->list.next; 1345 se = list_entry(lh, struct swap_extent, list); 1346 sis->curr_swap_extent = se; 1347 BUG_ON(se == start_se); /* It *must* be present */ 1348 } 1349} 1350 1351/* 1352 * Returns the page offset into bdev for the specified page's swap entry. 1353 */ 1354sector_t map_swap_page(struct page *page, struct block_device **bdev) 1355{ 1356 swp_entry_t entry; 1357 entry.val = page_private(page); 1358 return map_swap_entry(entry, bdev); 1359} 1360 1361/* 1362 * Free all of a swapdev's extent information 1363 */ 1364static void destroy_swap_extents(struct swap_info_struct *sis) 1365{ 1366 while (!list_empty(&sis->first_swap_extent.list)) { 1367 struct swap_extent *se; 1368 1369 se = list_entry(sis->first_swap_extent.list.next, 1370 struct swap_extent, list); 1371 list_del(&se->list); 1372 kfree(se); 1373 } 1374} 1375 1376/* 1377 * Add a block range (and the corresponding page range) into this swapdev's 1378 * extent list. The extent list is kept sorted in page order. 1379 * 1380 * This function rather assumes that it is called in ascending page order. 1381 */ 1382static int 1383add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, 1384 unsigned long nr_pages, sector_t start_block) 1385{ 1386 struct swap_extent *se; 1387 struct swap_extent *new_se; 1388 struct list_head *lh; 1389 1390 if (start_page == 0) { 1391 se = &sis->first_swap_extent; 1392 sis->curr_swap_extent = se; 1393 se->start_page = 0; 1394 se->nr_pages = nr_pages; 1395 se->start_block = start_block; 1396 return 1; 1397 } else { 1398 lh = sis->first_swap_extent.list.prev; /* Highest extent */ 1399 se = list_entry(lh, struct swap_extent, list); 1400 BUG_ON(se->start_page + se->nr_pages != start_page); 1401 if (se->start_block + se->nr_pages == start_block) { 1402 /* Merge it */ 1403 se->nr_pages += nr_pages; 1404 return 0; 1405 } 1406 } 1407 1408 /* 1409 * No merge. Insert a new extent, preserving ordering. 1410 */ 1411 new_se = kmalloc(sizeof(*se), GFP_KERNEL); 1412 if (new_se == NULL) 1413 return -ENOMEM; 1414 new_se->start_page = start_page; 1415 new_se->nr_pages = nr_pages; 1416 new_se->start_block = start_block; 1417 1418 list_add_tail(&new_se->list, &sis->first_swap_extent.list); 1419 return 1; 1420} 1421 1422/* 1423 * A `swap extent' is a simple thing which maps a contiguous range of pages 1424 * onto a contiguous range of disk blocks. An ordered list of swap extents 1425 * is built at swapon time and is then used at swap_writepage/swap_readpage 1426 * time for locating where on disk a page belongs. 1427 * 1428 * If the swapfile is an S_ISBLK block device, a single extent is installed. 1429 * This is done so that the main operating code can treat S_ISBLK and S_ISREG 1430 * swap files identically. 1431 * 1432 * Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap 1433 * extent list operates in PAGE_SIZE disk blocks. Both S_ISREG and S_ISBLK 1434 * swapfiles are handled *identically* after swapon time. 1435 * 1436 * For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks 1437 * and will parse them into an ordered extent list, in PAGE_SIZE chunks. If 1438 * some stray blocks are found which do not fall within the PAGE_SIZE alignment 1439 * requirements, they are simply tossed out - we will never use those blocks 1440 * for swapping. 1441 * 1442 * For S_ISREG swapfiles we set S_SWAPFILE across the life of the swapon. This 1443 * prevents root from shooting her foot off by ftruncating an in-use swapfile, 1444 * which will scribble on the fs. 1445 * 1446 * The amount of disk space which a single swap extent represents varies. 1447 * Typically it is in the 1-4 megabyte range. So we can have hundreds of 1448 * extents in the list. To avoid much list walking, we cache the previous 1449 * search location in `curr_swap_extent', and start new searches from there. 1450 * This is extremely effective. The average number of iterations in 1451 * map_swap_page() has been measured at about 0.3 per page. - akpm. 1452 */ 1453static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span) 1454{ 1455 struct inode *inode; 1456 unsigned blocks_per_page; 1457 unsigned long page_no; 1458 unsigned blkbits; 1459 sector_t probe_block; 1460 sector_t last_block; 1461 sector_t lowest_block = -1; 1462 sector_t highest_block = 0; 1463 int nr_extents = 0; 1464 int ret; 1465 1466 inode = sis->swap_file->f_mapping->host; 1467 if (S_ISBLK(inode->i_mode)) { 1468 ret = add_swap_extent(sis, 0, sis->max, 0); 1469 *span = sis->pages; 1470 goto out; 1471 } 1472 1473 blkbits = inode->i_blkbits; 1474 blocks_per_page = PAGE_SIZE >> blkbits; 1475 1476 /* 1477 * Map all the blocks into the extent list. This code doesn't try 1478 * to be very smart. 1479 */ 1480 probe_block = 0; 1481 page_no = 0; 1482 last_block = i_size_read(inode) >> blkbits; 1483 while ((probe_block + blocks_per_page) <= last_block && 1484 page_no < sis->max) { 1485 unsigned block_in_page; 1486 sector_t first_block; 1487 1488 first_block = bmap(inode, probe_block); 1489 if (first_block == 0) 1490 goto bad_bmap; 1491 1492 /* 1493 * It must be PAGE_SIZE aligned on-disk 1494 */ 1495 if (first_block & (blocks_per_page - 1)) { 1496 probe_block++; 1497 goto reprobe; 1498 } 1499 1500 for (block_in_page = 1; block_in_page < blocks_per_page; 1501 block_in_page++) { 1502 sector_t block; 1503 1504 block = bmap(inode, probe_block + block_in_page); 1505 if (block == 0) 1506 goto bad_bmap; 1507 if (block != first_block + block_in_page) { 1508 /* Discontiguity */ 1509 probe_block++; 1510 goto reprobe; 1511 } 1512 } 1513 1514 first_block >>= (PAGE_SHIFT - blkbits); 1515 if (page_no) { /* exclude the header page */ 1516 if (first_block < lowest_block) 1517 lowest_block = first_block; 1518 if (first_block > highest_block) 1519 highest_block = first_block; 1520 } 1521 1522 /* 1523 * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks 1524 */ 1525 ret = add_swap_extent(sis, page_no, 1, first_block); 1526 if (ret < 0) 1527 goto out; 1528 nr_extents += ret; 1529 page_no++; 1530 probe_block += blocks_per_page; 1531reprobe: 1532 continue; 1533 } 1534 ret = nr_extents; 1535 *span = 1 + highest_block - lowest_block; 1536 if (page_no == 0) 1537 page_no = 1; /* force Empty message */ 1538 sis->max = page_no; 1539 sis->pages = page_no - 1; 1540 sis->highest_bit = page_no - 1; 1541out: 1542 return ret; 1543bad_bmap: 1544 printk(KERN_ERR "swapon: swapfile has holes\n"); 1545 ret = -EINVAL; 1546 goto out; 1547} 1548 1549SYSCALL_DEFINE1(swapoff, const char __user *, specialfile) 1550{ 1551 struct swap_info_struct *p = NULL; 1552 unsigned char *swap_map; 1553 struct file *swap_file, *victim; 1554 struct address_space *mapping; 1555 struct inode *inode; 1556 char *pathname; 1557 int i, type, prev; 1558 int err; 1559 1560 if (!capable(CAP_SYS_ADMIN)) 1561 return -EPERM; 1562 1563 pathname = getname(specialfile); 1564 err = PTR_ERR(pathname); 1565 if (IS_ERR(pathname)) 1566 goto out; 1567 1568 victim = filp_open(pathname, O_RDWR|O_LARGEFILE, 0); 1569 putname(pathname); 1570 err = PTR_ERR(victim); 1571 if (IS_ERR(victim)) 1572 goto out; 1573 1574 mapping = victim->f_mapping; 1575 prev = -1; 1576 spin_lock(&swap_lock); 1577 for (type = swap_list.head; type >= 0; type = swap_info[type]->next) { 1578 p = swap_info[type]; 1579 if (p->flags & SWP_WRITEOK) { 1580 if (p->swap_file->f_mapping == mapping) 1581 break; 1582 } 1583 prev = type; 1584 } 1585 if (type < 0) { 1586 err = -EINVAL; 1587 spin_unlock(&swap_lock); 1588 goto out_dput; 1589 } 1590 if (!security_vm_enough_memory(p->pages)) 1591 vm_unacct_memory(p->pages); 1592 else { 1593 err = -ENOMEM; 1594 spin_unlock(&swap_lock); 1595 goto out_dput; 1596 } 1597 if (prev < 0) 1598 swap_list.head = p->next; 1599 else 1600 swap_info[prev]->next = p->next; 1601 if (type == swap_list.next) { 1602 /* just pick something that's safe... */ 1603 swap_list.next = swap_list.head; 1604 } 1605 if (p->prio < 0) { 1606 for (i = p->next; i >= 0; i = swap_info[i]->next) 1607 swap_info[i]->prio = p->prio--; 1608 least_priority++; 1609 } 1610 nr_swap_pages -= p->pages; 1611 total_swap_pages -= p->pages; 1612 p->flags &= ~SWP_WRITEOK; 1613 spin_unlock(&swap_lock); 1614 1615 current->flags |= PF_OOM_ORIGIN; 1616 err = try_to_unuse(type); 1617 current->flags &= ~PF_OOM_ORIGIN; 1618 1619 if (err) { 1620 /* re-insert swap space back into swap_list */ 1621 spin_lock(&swap_lock); 1622 if (p->prio < 0) 1623 p->prio = --least_priority; 1624 prev = -1; 1625 for (i = swap_list.head; i >= 0; i = swap_info[i]->next) { 1626 if (p->prio >= swap_info[i]->prio) 1627 break; 1628 prev = i; 1629 } 1630 p->next = i; 1631 if (prev < 0) 1632 swap_list.head = swap_list.next = type; 1633 else 1634 swap_info[prev]->next = type; 1635 nr_swap_pages += p->pages; 1636 total_swap_pages += p->pages; 1637 p->flags |= SWP_WRITEOK; 1638 spin_unlock(&swap_lock); 1639 goto out_dput; 1640 } 1641 1642 /* wait for any unplug function to finish */ 1643 down_write(&swap_unplug_sem); 1644 up_write(&swap_unplug_sem); 1645 1646 destroy_swap_extents(p); 1647 if (p->flags & SWP_CONTINUED) 1648 free_swap_count_continuations(p); 1649 1650 mutex_lock(&swapon_mutex); 1651 spin_lock(&swap_lock); 1652 drain_mmlist(); 1653 1654 /* wait for anyone still in scan_swap_map */ 1655 p->highest_bit = 0; /* cuts scans short */ 1656 while (p->flags >= SWP_SCANNING) { 1657 spin_unlock(&swap_lock); 1658 schedule_timeout_uninterruptible(1); 1659 spin_lock(&swap_lock); 1660 } 1661 1662 swap_file = p->swap_file; 1663 p->swap_file = NULL; 1664 p->max = 0; 1665 swap_map = p->swap_map; 1666 p->swap_map = NULL; 1667 p->flags = 0; 1668 spin_unlock(&swap_lock); 1669 mutex_unlock(&swapon_mutex); 1670 vfree(swap_map); 1671 /* Destroy swap account informatin */ 1672 swap_cgroup_swapoff(type); 1673 1674 inode = mapping->host; 1675 if (S_ISBLK(inode->i_mode)) { 1676 struct block_device *bdev = I_BDEV(inode); 1677 set_blocksize(bdev, p->old_block_size); 1678 bd_release(bdev); 1679 } else { 1680 mutex_lock(&inode->i_mutex); 1681 inode->i_flags &= ~S_SWAPFILE; 1682 mutex_unlock(&inode->i_mutex); 1683 } 1684 filp_close(swap_file, NULL); 1685 err = 0; 1686 1687out_dput: 1688 filp_close(victim, NULL); 1689out: 1690 return err; 1691} 1692 1693#ifdef CONFIG_PROC_FS 1694/* iterator */ 1695static void *swap_start(struct seq_file *swap, loff_t *pos) 1696{ 1697 struct swap_info_struct *si; 1698 int type; 1699 loff_t l = *pos; 1700 1701 mutex_lock(&swapon_mutex); 1702 1703 if (!l) 1704 return SEQ_START_TOKEN; 1705 1706 for (type = 0; type < nr_swapfiles; type++) { 1707 smp_rmb(); /* read nr_swapfiles before swap_info[type] */ 1708 si = swap_info[type]; 1709 if (!(si->flags & SWP_USED) || !si->swap_map) 1710 continue; 1711 if (!--l) 1712 return si; 1713 } 1714 1715 return NULL; 1716} 1717 1718static void *swap_next(struct seq_file *swap, void *v, loff_t *pos) 1719{ 1720 struct swap_info_struct *si = v; 1721 int type; 1722 1723 if (v == SEQ_START_TOKEN) 1724 type = 0; 1725 else 1726 type = si->type + 1; 1727 1728 for (; type < nr_swapfiles; type++) { 1729 smp_rmb(); /* read nr_swapfiles before swap_info[type] */ 1730 si = swap_info[type]; 1731 if (!(si->flags & SWP_USED) || !si->swap_map) 1732 continue; 1733 ++*pos; 1734 return si; 1735 } 1736 1737 return NULL; 1738} 1739 1740static void swap_stop(struct seq_file *swap, void *v) 1741{ 1742 mutex_unlock(&swapon_mutex); 1743} 1744 1745static int swap_show(struct seq_file *swap, void *v) 1746{ 1747 struct swap_info_struct *si = v; 1748 struct file *file; 1749 int len; 1750 1751 if (si == SEQ_START_TOKEN) { 1752 seq_puts(swap,"Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n"); 1753 return 0; 1754 } 1755 1756 file = si->swap_file; 1757 len = seq_path(swap, &file->f_path, " \t\n\\"); 1758 seq_printf(swap, "%*s%s\t%u\t%u\t%d\n", 1759 len < 40 ? 40 - len : 1, " ", 1760 S_ISBLK(file->f_path.dentry->d_inode->i_mode) ? 1761 "partition" : "file\t", 1762 si->pages << (PAGE_SHIFT - 10), 1763 si->inuse_pages << (PAGE_SHIFT - 10), 1764 si->prio); 1765 return 0; 1766} 1767 1768static const struct seq_operations swaps_op = { 1769 .start = swap_start, 1770 .next = swap_next, 1771 .stop = swap_stop, 1772 .show = swap_show 1773}; 1774 1775static int swaps_open(struct inode *inode, struct file *file) 1776{ 1777 return seq_open(file, &swaps_op); 1778} 1779 1780static const struct file_operations proc_swaps_operations = { 1781 .open = swaps_open, 1782 .read = seq_read, 1783 .llseek = seq_lseek, 1784 .release = seq_release, 1785}; 1786 1787static int __init procswaps_init(void) 1788{ 1789 proc_create("swaps", 0, NULL, &proc_swaps_operations); 1790 return 0; 1791} 1792__initcall(procswaps_init); 1793#endif /* CONFIG_PROC_FS */ 1794 1795#ifdef MAX_SWAPFILES_CHECK 1796static int __init max_swapfiles_check(void) 1797{ 1798 MAX_SWAPFILES_CHECK(); 1799 return 0; 1800} 1801late_initcall(max_swapfiles_check); 1802#endif 1803 1804/* 1805 * Written 01/25/92 by Simmule Turner, heavily changed by Linus. 1806 * 1807 * The swapon system call 1808 */ 1809SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) 1810{ 1811 struct swap_info_struct *p; 1812 char *name = NULL; 1813 struct block_device *bdev = NULL; 1814 struct file *swap_file = NULL; 1815 struct address_space *mapping; 1816 unsigned int type; 1817 int i, prev; 1818 int error; 1819 union swap_header *swap_header; 1820 unsigned int nr_good_pages; 1821 int nr_extents = 0; 1822 sector_t span; 1823 unsigned long maxpages; 1824 unsigned long swapfilepages; 1825 unsigned char *swap_map = NULL; 1826 struct page *page = NULL; 1827 struct inode *inode = NULL; 1828 int did_down = 0; 1829 1830 if (!capable(CAP_SYS_ADMIN)) 1831 return -EPERM; 1832 1833 p = kzalloc(sizeof(*p), GFP_KERNEL); 1834 if (!p) 1835 return -ENOMEM; 1836 1837 spin_lock(&swap_lock); 1838 for (type = 0; type < nr_swapfiles; type++) { 1839 if (!(swap_info[type]->flags & SWP_USED)) 1840 break; 1841 } 1842 error = -EPERM; 1843 if (type >= MAX_SWAPFILES) { 1844 spin_unlock(&swap_lock); 1845 kfree(p); 1846 goto out; 1847 } 1848 if (type >= nr_swapfiles) { 1849 p->type = type; 1850 swap_info[type] = p; 1851 /* 1852 * Write swap_info[type] before nr_swapfiles, in case a 1853 * racing procfs swap_start() or swap_next() is reading them. 1854 * (We never shrink nr_swapfiles, we never free this entry.) 1855 */ 1856 smp_wmb(); 1857 nr_swapfiles++; 1858 } else { 1859 kfree(p); 1860 p = swap_info[type]; 1861 /* 1862 * Do not memset this entry: a racing procfs swap_next() 1863 * would be relying on p->type to remain valid. 1864 */ 1865 } 1866 INIT_LIST_HEAD(&p->first_swap_extent.list); 1867 p->flags = SWP_USED; 1868 p->next = -1; 1869 spin_unlock(&swap_lock); 1870 1871 name = getname(specialfile); 1872 error = PTR_ERR(name); 1873 if (IS_ERR(name)) { 1874 name = NULL; 1875 goto bad_swap_2; 1876 } 1877 swap_file = filp_open(name, O_RDWR|O_LARGEFILE, 0); 1878 error = PTR_ERR(swap_file); 1879 if (IS_ERR(swap_file)) { 1880 swap_file = NULL; 1881 goto bad_swap_2; 1882 } 1883 1884 p->swap_file = swap_file; 1885 mapping = swap_file->f_mapping; 1886 inode = mapping->host; 1887 1888 error = -EBUSY; 1889 for (i = 0; i < nr_swapfiles; i++) { 1890 struct swap_info_struct *q = swap_info[i]; 1891 1892 if (i == type || !q->swap_file) 1893 continue; 1894 if (mapping == q->swap_file->f_mapping) 1895 goto bad_swap; 1896 } 1897 1898 error = -EINVAL; 1899 if (S_ISBLK(inode->i_mode)) { 1900 bdev = I_BDEV(inode); 1901 error = bd_claim(bdev, sys_swapon); 1902 if (error < 0) { 1903 bdev = NULL; 1904 error = -EINVAL; 1905 goto bad_swap; 1906 } 1907 p->old_block_size = block_size(bdev); 1908 error = set_blocksize(bdev, PAGE_SIZE); 1909 if (error < 0) 1910 goto bad_swap; 1911 p->bdev = bdev; 1912 p->flags |= SWP_BLKDEV; 1913 } else if (S_ISREG(inode->i_mode)) { 1914 p->bdev = inode->i_sb->s_bdev; 1915 mutex_lock(&inode->i_mutex); 1916 did_down = 1; 1917 if (IS_SWAPFILE(inode)) { 1918 error = -EBUSY; 1919 goto bad_swap; 1920 } 1921 } else { 1922 goto bad_swap; 1923 } 1924 1925 swapfilepages = i_size_read(inode) >> PAGE_SHIFT; 1926 1927 /* 1928 * Read the swap header. 1929 */ 1930 if (!mapping->a_ops->readpage) { 1931 error = -EINVAL; 1932 goto bad_swap; 1933 } 1934 page = read_mapping_page(mapping, 0, swap_file); 1935 if (IS_ERR(page)) { 1936 error = PTR_ERR(page); 1937 goto bad_swap; 1938 } 1939 swap_header = kmap(page); 1940 1941 if (memcmp("SWAPSPACE2", swap_header->magic.magic, 10)) { 1942 printk(KERN_ERR "Unable to find swap-space signature\n"); 1943 error = -EINVAL; 1944 goto bad_swap; 1945 } 1946 1947 /* swap partition endianess hack... */ 1948 if (swab32(swap_header->info.version) == 1) { 1949 swab32s(&swap_header->info.version); 1950 swab32s(&swap_header->info.last_page); 1951 swab32s(&swap_header->info.nr_badpages); 1952 for (i = 0; i < swap_header->info.nr_badpages; i++) 1953 swab32s(&swap_header->info.badpages[i]); 1954 } 1955 /* Check the swap header's sub-version */ 1956 if (swap_header->info.version != 1) { 1957 printk(KERN_WARNING 1958 "Unable to handle swap header version %d\n", 1959 swap_header->info.version); 1960 error = -EINVAL; 1961 goto bad_swap; 1962 } 1963 1964 p->lowest_bit = 1; 1965 p->cluster_next = 1; 1966 p->cluster_nr = 0; 1967 1968 /* 1969 * Find out how many pages are allowed for a single swap 1970 * device. There are two limiting factors: 1) the number of 1971 * bits for the swap offset in the swp_entry_t type and 1972 * 2) the number of bits in the a swap pte as defined by 1973 * the different architectures. In order to find the 1974 * largest possible bit mask a swap entry with swap type 0 1975 * and swap offset ~0UL is created, encoded to a swap pte, 1976 * decoded to a swp_entry_t again and finally the swap 1977 * offset is extracted. This will mask all the bits from 1978 * the initial ~0UL mask that can't be encoded in either 1979 * the swp_entry_t or the architecture definition of a 1980 * swap pte. 1981 */ 1982 maxpages = swp_offset(pte_to_swp_entry( 1983 swp_entry_to_pte(swp_entry(0, ~0UL)))) + 1; 1984 if (maxpages > swap_header->info.last_page) { 1985 maxpages = swap_header->info.last_page + 1; 1986 /* p->max is an unsigned int: don't overflow it */ 1987 if ((unsigned int)maxpages == 0) 1988 maxpages = UINT_MAX; 1989 } 1990 p->highest_bit = maxpages - 1; 1991 1992 error = -EINVAL; 1993 if (!maxpages) 1994 goto bad_swap; 1995 if (swapfilepages && maxpages > swapfilepages) { 1996 printk(KERN_WARNING 1997 "Swap area shorter than signature indicates\n"); 1998 goto bad_swap; 1999 } 2000 if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode)) 2001 goto bad_swap; 2002 if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES) 2003 goto bad_swap; 2004 2005 /* OK, set up the swap map and apply the bad block list */ 2006 swap_map = vmalloc(maxpages); 2007 if (!swap_map) { 2008 error = -ENOMEM; 2009 goto bad_swap; 2010 } 2011 2012 memset(swap_map, 0, maxpages); 2013 nr_good_pages = maxpages - 1; /* omit header page */ 2014 2015 for (i = 0; i < swap_header->info.nr_badpages; i++) { 2016 unsigned int page_nr = swap_header->info.badpages[i]; 2017 if (page_nr == 0 || page_nr > swap_header->info.last_page) { 2018 error = -EINVAL; 2019 goto bad_swap; 2020 } 2021 if (page_nr < maxpages) { 2022 swap_map[page_nr] = SWAP_MAP_BAD; 2023 nr_good_pages--; 2024 } 2025 } 2026 2027 error = swap_cgroup_swapon(type, maxpages); 2028 if (error) 2029 goto bad_swap; 2030 2031 if (nr_good_pages) { 2032 swap_map[0] = SWAP_MAP_BAD; 2033 p->max = maxpages; 2034 p->pages = nr_good_pages; 2035 nr_extents = setup_swap_extents(p, &span); 2036 if (nr_extents < 0) { 2037 error = nr_extents; 2038 goto bad_swap; 2039 } 2040 nr_good_pages = p->pages; 2041 } 2042 if (!nr_good_pages) { 2043 printk(KERN_WARNING "Empty swap-file\n"); 2044 error = -EINVAL; 2045 goto bad_swap; 2046 } 2047 2048 if (p->bdev) { 2049 if (blk_queue_nonrot(bdev_get_queue(p->bdev))) { 2050 p->flags |= SWP_SOLIDSTATE; 2051 p->cluster_next = 1 + (random32() % p->highest_bit); 2052 } 2053 if (discard_swap(p) == 0) 2054 p->flags |= SWP_DISCARDABLE; 2055 } 2056 2057 mutex_lock(&swapon_mutex); 2058 spin_lock(&swap_lock); 2059 if (swap_flags & SWAP_FLAG_PREFER) 2060 p->prio = 2061 (swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT; 2062 else 2063 p->prio = --least_priority; 2064 p->swap_map = swap_map; 2065 p->flags |= SWP_WRITEOK; 2066 nr_swap_pages += nr_good_pages; 2067 total_swap_pages += nr_good_pages; 2068 2069 printk(KERN_INFO "Adding %uk swap on %s. " 2070 "Priority:%d extents:%d across:%lluk %s%s\n", 2071 nr_good_pages<<(PAGE_SHIFT-10), name, p->prio, 2072 nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10), 2073 (p->flags & SWP_SOLIDSTATE) ? "SS" : "", 2074 (p->flags & SWP_DISCARDABLE) ? "D" : ""); 2075 2076 /* insert swap space into swap_list: */ 2077 prev = -1; 2078 for (i = swap_list.head; i >= 0; i = swap_info[i]->next) { 2079 if (p->prio >= swap_info[i]->prio) 2080 break; 2081 prev = i; 2082 } 2083 p->next = i; 2084 if (prev < 0) 2085 swap_list.head = swap_list.next = type; 2086 else 2087 swap_info[prev]->next = type; 2088 spin_unlock(&swap_lock); 2089 mutex_unlock(&swapon_mutex); 2090 error = 0; 2091 goto out; 2092bad_swap: 2093 if (bdev) { 2094 set_blocksize(bdev, p->old_block_size); 2095 bd_release(bdev); 2096 } 2097 destroy_swap_extents(p); 2098 swap_cgroup_swapoff(type); 2099bad_swap_2: 2100 spin_lock(&swap_lock); 2101 p->swap_file = NULL; 2102 p->flags = 0; 2103 spin_unlock(&swap_lock); 2104 vfree(swap_map); 2105 if (swap_file) 2106 filp_close(swap_file, NULL); 2107out: 2108 if (page && !IS_ERR(page)) { 2109 kunmap(page); 2110 page_cache_release(page); 2111 } 2112 if (name) 2113 putname(name); 2114 if (did_down) { 2115 if (!error) 2116 inode->i_flags |= S_SWAPFILE; 2117 mutex_unlock(&inode->i_mutex); 2118 } 2119 return error; 2120} 2121 2122void si_swapinfo(struct sysinfo *val) 2123{ 2124 unsigned int type; 2125 unsigned long nr_to_be_unused = 0; 2126 2127 spin_lock(&swap_lock); 2128 for (type = 0; type < nr_swapfiles; type++) { 2129 struct swap_info_struct *si = swap_info[type]; 2130 2131 if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK)) 2132 nr_to_be_unused += si->inuse_pages; 2133 } 2134 val->freeswap = nr_swap_pages + nr_to_be_unused; 2135 val->totalswap = total_swap_pages + nr_to_be_unused; 2136 spin_unlock(&swap_lock); 2137} 2138 2139/* 2140 * Verify that a swap entry is valid and increment its swap map count. 2141 * 2142 * Returns error code in following case. 2143 * - success -> 0 2144 * - swp_entry is invalid -> EINVAL 2145 * - swp_entry is migration entry -> EINVAL 2146 * - swap-cache reference is requested but there is already one. -> EEXIST 2147 * - swap-cache reference is requested but the entry is not used. -> ENOENT 2148 * - swap-mapped reference requested but needs continued swap count. -> ENOMEM 2149 */ 2150static int __swap_duplicate(swp_entry_t entry, unsigned char usage) 2151{ 2152 struct swap_info_struct *p; 2153 unsigned long offset, type; 2154 unsigned char count; 2155 unsigned char has_cache; 2156 int err = -EINVAL; 2157 2158 if (non_swap_entry(entry)) 2159 goto out; 2160 2161 type = swp_type(entry); 2162 if (type >= nr_swapfiles) 2163 goto bad_file; 2164 p = swap_info[type]; 2165 offset = swp_offset(entry); 2166 2167 spin_lock(&swap_lock); 2168 if (unlikely(offset >= p->max)) 2169 goto unlock_out; 2170 2171 count = p->swap_map[offset]; 2172 has_cache = count & SWAP_HAS_CACHE; 2173 count &= ~SWAP_HAS_CACHE; 2174 err = 0; 2175 2176 if (usage == SWAP_HAS_CACHE) { 2177 2178 /* set SWAP_HAS_CACHE if there is no cache and entry is used */ 2179 if (!has_cache && count) 2180 has_cache = SWAP_HAS_CACHE; 2181 else if (has_cache) /* someone else added cache */ 2182 err = -EEXIST; 2183 else /* no users remaining */ 2184 err = -ENOENT; 2185 2186 } else if (count || has_cache) { 2187 2188 if ((count & ~COUNT_CONTINUED) < SWAP_MAP_MAX) 2189 count += usage; 2190 else if ((count & ~COUNT_CONTINUED) > SWAP_MAP_MAX) 2191 err = -EINVAL; 2192 else if (swap_count_continued(p, offset, count)) 2193 count = COUNT_CONTINUED; 2194 else 2195 err = -ENOMEM; 2196 } else 2197 err = -ENOENT; /* unused swap entry */ 2198 2199 p->swap_map[offset] = count | has_cache; 2200 2201unlock_out: 2202 spin_unlock(&swap_lock); 2203out: 2204 return err; 2205 2206bad_file: 2207 printk(KERN_ERR "swap_dup: %s%08lx\n", Bad_file, entry.val); 2208 goto out; 2209} 2210 2211/* 2212 * Help swapoff by noting that swap entry belongs to shmem/tmpfs 2213 * (in which case its reference count is never incremented). 2214 */ 2215void swap_shmem_alloc(swp_entry_t entry) 2216{ 2217 __swap_duplicate(entry, SWAP_MAP_SHMEM); 2218} 2219 2220/* 2221 * Increase reference count of swap entry by 1. 2222 * Returns 0 for success, or -ENOMEM if a swap_count_continuation is required 2223 * but could not be atomically allocated. Returns 0, just as if it succeeded, 2224 * if __swap_duplicate() fails for another reason (-EINVAL or -ENOENT), which 2225 * might occur if a page table entry has got corrupted. 2226 */ 2227int swap_duplicate(swp_entry_t entry) 2228{ 2229 int err = 0; 2230 2231 while (!err && __swap_duplicate(entry, 1) == -ENOMEM) 2232 err = add_swap_count_continuation(entry, GFP_ATOMIC); 2233 return err; 2234} 2235 2236/* 2237 * @entry: swap entry for which we allocate swap cache. 2238 * 2239 * Called when allocating swap cache for existing swap entry, 2240 * This can return error codes. Returns 0 at success. 2241 * -EBUSY means there is a swap cache. 2242 * Note: return code is different from swap_duplicate(). 2243 */ 2244int swapcache_prepare(swp_entry_t entry) 2245{ 2246 return __swap_duplicate(entry, SWAP_HAS_CACHE); 2247} 2248 2249/* 2250 * swap_lock prevents swap_map being freed. Don't grab an extra 2251 * reference on the swaphandle, it doesn't matter if it becomes unused. 2252 */ 2253int valid_swaphandles(swp_entry_t entry, unsigned long *offset) 2254{ 2255 struct swap_info_struct *si; 2256 int our_page_cluster = page_cluster; 2257 pgoff_t target, toff; 2258 pgoff_t base, end; 2259 int nr_pages = 0; 2260 2261 if (!our_page_cluster) /* no readahead */ 2262 return 0; 2263 2264 si = swap_info[swp_type(entry)]; 2265 target = swp_offset(entry); 2266 base = (target >> our_page_cluster) << our_page_cluster; 2267 end = base + (1 << our_page_cluster); 2268 if (!base) /* first page is swap header */ 2269 base++; 2270 2271 spin_lock(&swap_lock); 2272 if (end > si->max) /* don't go beyond end of map */ 2273 end = si->max; 2274 2275 /* Count contiguous allocated slots above our target */ 2276 for (toff = target; ++toff < end; nr_pages++) { 2277 /* Don't read in free or bad pages */ 2278 if (!si->swap_map[toff]) 2279 break; 2280 if (swap_count(si->swap_map[toff]) == SWAP_MAP_BAD) 2281 break; 2282 } 2283 /* Count contiguous allocated slots below our target */ 2284 for (toff = target; --toff >= base; nr_pages++) { 2285 /* Don't read in free or bad pages */ 2286 if (!si->swap_map[toff]) 2287 break; 2288 if (swap_count(si->swap_map[toff]) == SWAP_MAP_BAD) 2289 break; 2290 } 2291 spin_unlock(&swap_lock); 2292 2293 /* 2294 * Indicate starting offset, and return number of pages to get: 2295 * if only 1, say 0, since there's then no readahead to be done. 2296 */ 2297 *offset = ++toff; 2298 return nr_pages? ++nr_pages: 0; 2299} 2300 2301/* 2302 * add_swap_count_continuation - called when a swap count is duplicated 2303 * beyond SWAP_MAP_MAX, it allocates a new page and links that to the entry's 2304 * page of the original vmalloc'ed swap_map, to hold the continuation count 2305 * (for that entry and for its neighbouring PAGE_SIZE swap entries). Called 2306 * again when count is duplicated beyond SWAP_MAP_MAX * SWAP_CONT_MAX, etc. 2307 * 2308 * These continuation pages are seldom referenced: the common paths all work 2309 * on the original swap_map, only referring to a continuation page when the 2310 * low "digit" of a count is incremented or decremented through SWAP_MAP_MAX. 2311 * 2312 * add_swap_count_continuation(, GFP_ATOMIC) can be called while holding 2313 * page table locks; if it fails, add_swap_count_continuation(, GFP_KERNEL) 2314 * can be called after dropping locks. 2315 */ 2316int add_swap_count_continuation(swp_entry_t entry, gfp_t gfp_mask) 2317{ 2318 struct swap_info_struct *si; 2319 struct page *head; 2320 struct page *page; 2321 struct page *list_page; 2322 pgoff_t offset; 2323 unsigned char count; 2324 2325 /* 2326 * When debugging, it's easier to use __GFP_ZERO here; but it's better 2327 * for latency not to zero a page while GFP_ATOMIC and holding locks. 2328 */ 2329 page = alloc_page(gfp_mask | __GFP_HIGHMEM); 2330 2331 si = swap_info_get(entry); 2332 if (!si) { 2333 /* 2334 * An acceptable race has occurred since the failing 2335 * __swap_duplicate(): the swap entry has been freed, 2336 * perhaps even the whole swap_map cleared for swapoff. 2337 */ 2338 goto outer; 2339 } 2340 2341 offset = swp_offset(entry); 2342 count = si->swap_map[offset] & ~SWAP_HAS_CACHE; 2343 2344 if ((count & ~COUNT_CONTINUED) != SWAP_MAP_MAX) { 2345 /* 2346 * The higher the swap count, the more likely it is that tasks 2347 * will race to add swap count continuation: we need to avoid 2348 * over-provisioning. 2349 */ 2350 goto out; 2351 } 2352 2353 if (!page) { 2354 spin_unlock(&swap_lock); 2355 return -ENOMEM; 2356 } 2357 2358 /* 2359 * We are fortunate that although vmalloc_to_page uses pte_offset_map, 2360 * no architecture is using highmem pages for kernel pagetables: so it 2361 * will not corrupt the GFP_ATOMIC caller's atomic pagetable kmaps. 2362 */ 2363 head = vmalloc_to_page(si->swap_map + offset); 2364 offset &= ~PAGE_MASK; 2365 2366 /* 2367 * Page allocation does not initialize the page's lru field, 2368 * but it does always reset its private field. 2369 */ 2370 if (!page_private(head)) { 2371 BUG_ON(count & COUNT_CONTINUED); 2372 INIT_LIST_HEAD(&head->lru); 2373 set_page_private(head, SWP_CONTINUED); 2374 si->flags |= SWP_CONTINUED; 2375 } 2376 2377 list_for_each_entry(list_page, &head->lru, lru) { 2378 unsigned char *map; 2379 2380 /* 2381 * If the previous map said no continuation, but we've found 2382 * a continuation page, free our allocation and use this one. 2383 */ 2384 if (!(count & COUNT_CONTINUED)) 2385 goto out; 2386 2387 map = kmap_atomic(list_page, KM_USER0) + offset; 2388 count = *map; 2389 kunmap_atomic(map, KM_USER0); 2390 2391 /* 2392 * If this continuation count now has some space in it, 2393 * free our allocation and use this one. 2394 */ 2395 if ((count & ~COUNT_CONTINUED) != SWAP_CONT_MAX) 2396 goto out; 2397 } 2398 2399 list_add_tail(&page->lru, &head->lru); 2400 page = NULL; /* now it's attached, don't free it */ 2401out: 2402 spin_unlock(&swap_lock); 2403outer: 2404 if (page) 2405 __free_page(page); 2406 return 0; 2407} 2408 2409/* 2410 * swap_count_continued - when the original swap_map count is incremented 2411 * from SWAP_MAP_MAX, check if there is already a continuation page to carry 2412 * into, carry if so, or else fail until a new continuation page is allocated; 2413 * when the original swap_map count is decremented from 0 with continuation, 2414 * borrow from the continuation and report whether it still holds more. 2415 * Called while __swap_duplicate() or swap_entry_free() holds swap_lock. 2416 */ 2417static bool swap_count_continued(struct swap_info_struct *si, 2418 pgoff_t offset, unsigned char count) 2419{ 2420 struct page *head; 2421 struct page *page; 2422 unsigned char *map; 2423 2424 head = vmalloc_to_page(si->swap_map + offset); 2425 if (page_private(head) != SWP_CONTINUED) { 2426 BUG_ON(count & COUNT_CONTINUED); 2427 return false; /* need to add count continuation */ 2428 } 2429 2430 offset &= ~PAGE_MASK; 2431 page = list_entry(head->lru.next, struct page, lru); 2432 map = kmap_atomic(page, KM_USER0) + offset; 2433 2434 if (count == SWAP_MAP_MAX) /* initial increment from swap_map */ 2435 goto init_map; /* jump over SWAP_CONT_MAX checks */ 2436 2437 if (count == (SWAP_MAP_MAX | COUNT_CONTINUED)) { /* incrementing */ 2438 /* 2439 * Think of how you add 1 to 999 2440 */ 2441 while (*map == (SWAP_CONT_MAX | COUNT_CONTINUED)) { 2442 kunmap_atomic(map, KM_USER0); 2443 page = list_entry(page->lru.next, struct page, lru); 2444 BUG_ON(page == head); 2445 map = kmap_atomic(page, KM_USER0) + offset; 2446 } 2447 if (*map == SWAP_CONT_MAX) { 2448 kunmap_atomic(map, KM_USER0); 2449 page = list_entry(page->lru.next, struct page, lru); 2450 if (page == head) 2451 return false; /* add count continuation */ 2452 map = kmap_atomic(page, KM_USER0) + offset; 2453init_map: *map = 0; /* we didn't zero the page */ 2454 } 2455 *map += 1; 2456 kunmap_atomic(map, KM_USER0); 2457 page = list_entry(page->lru.prev, struct page, lru); 2458 while (page != head) { 2459 map = kmap_atomic(page, KM_USER0) + offset; 2460 *map = COUNT_CONTINUED; 2461 kunmap_atomic(map, KM_USER0); 2462 page = list_entry(page->lru.prev, struct page, lru); 2463 } 2464 return true; /* incremented */ 2465 2466 } else { /* decrementing */ 2467 /* 2468 * Think of how you subtract 1 from 1000 2469 */ 2470 BUG_ON(count != COUNT_CONTINUED); 2471 while (*map == COUNT_CONTINUED) { 2472 kunmap_atomic(map, KM_USER0); 2473 page = list_entry(page->lru.next, struct page, lru); 2474 BUG_ON(page == head); 2475 map = kmap_atomic(page, KM_USER0) + offset; 2476 } 2477 BUG_ON(*map == 0); 2478 *map -= 1; 2479 if (*map == 0) 2480 count = 0; 2481 kunmap_atomic(map, KM_USER0); 2482 page = list_entry(page->lru.prev, struct page, lru); 2483 while (page != head) { 2484 map = kmap_atomic(page, KM_USER0) + offset; 2485 *map = SWAP_CONT_MAX | count; 2486 count = COUNT_CONTINUED; 2487 kunmap_atomic(map, KM_USER0); 2488 page = list_entry(page->lru.prev, struct page, lru); 2489 } 2490 return count == COUNT_CONTINUED; 2491 } 2492} 2493 2494/* 2495 * free_swap_count_continuations - swapoff free all the continuation pages 2496 * appended to the swap_map, after swap_map is quiesced, before vfree'ing it. 2497 */ 2498static void free_swap_count_continuations(struct swap_info_struct *si) 2499{ 2500 pgoff_t offset; 2501 2502 for (offset = 0; offset < si->max; offset += PAGE_SIZE) { 2503 struct page *head; 2504 head = vmalloc_to_page(si->swap_map + offset); 2505 if (page_private(head)) { 2506 struct list_head *this, *next; 2507 list_for_each_safe(this, next, &head->lru) { 2508 struct page *page; 2509 page = list_entry(this, struct page, lru); 2510 list_del(this); 2511 __free_page(page); 2512 } 2513 } 2514 } 2515} 2516