swapfile.c revision 20137a490f397d9c01fc9fadd83a8d198bda4477
1/* 2 * linux/mm/swapfile.c 3 * 4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 5 * Swap reorganised 29.12.95, Stephen Tweedie 6 */ 7 8#include <linux/mm.h> 9#include <linux/hugetlb.h> 10#include <linux/mman.h> 11#include <linux/slab.h> 12#include <linux/kernel_stat.h> 13#include <linux/swap.h> 14#include <linux/vmalloc.h> 15#include <linux/pagemap.h> 16#include <linux/namei.h> 17#include <linux/shm.h> 18#include <linux/blkdev.h> 19#include <linux/random.h> 20#include <linux/writeback.h> 21#include <linux/proc_fs.h> 22#include <linux/seq_file.h> 23#include <linux/init.h> 24#include <linux/module.h> 25#include <linux/rmap.h> 26#include <linux/security.h> 27#include <linux/backing-dev.h> 28#include <linux/mutex.h> 29#include <linux/capability.h> 30#include <linux/syscalls.h> 31#include <linux/memcontrol.h> 32 33#include <asm/pgtable.h> 34#include <asm/tlbflush.h> 35#include <linux/swapops.h> 36 37static DEFINE_SPINLOCK(swap_lock); 38static unsigned int nr_swapfiles; 39long nr_swap_pages; 40long total_swap_pages; 41static int swap_overflow; 42static int least_priority; 43 44static const char Bad_file[] = "Bad swap file entry "; 45static const char Unused_file[] = "Unused swap file entry "; 46static const char Bad_offset[] = "Bad swap offset entry "; 47static const char Unused_offset[] = "Unused swap offset entry "; 48 49static struct swap_list_t swap_list = {-1, -1}; 50 51static struct swap_info_struct swap_info[MAX_SWAPFILES]; 52 53static DEFINE_MUTEX(swapon_mutex); 54 55/* 56 * We need this because the bdev->unplug_fn can sleep and we cannot 57 * hold swap_lock while calling the unplug_fn. And swap_lock 58 * cannot be turned into a mutex. 59 */ 60static DECLARE_RWSEM(swap_unplug_sem); 61 62void swap_unplug_io_fn(struct backing_dev_info *unused_bdi, struct page *page) 63{ 64 swp_entry_t entry; 65 66 down_read(&swap_unplug_sem); 67 entry.val = page_private(page); 68 if (PageSwapCache(page)) { 69 struct block_device *bdev = swap_info[swp_type(entry)].bdev; 70 struct backing_dev_info *bdi; 71 72 /* 73 * If the page is removed from swapcache from under us (with a 74 * racy try_to_unuse/swapoff) we need an additional reference 75 * count to avoid reading garbage from page_private(page) above. 76 * If the WARN_ON triggers during a swapoff it maybe the race 77 * condition and it's harmless. However if it triggers without 78 * swapoff it signals a problem. 79 */ 80 WARN_ON(page_count(page) <= 1); 81 82 bdi = bdev->bd_inode->i_mapping->backing_dev_info; 83 blk_run_backing_dev(bdi, page); 84 } 85 up_read(&swap_unplug_sem); 86} 87 88/* 89 * swapon tell device that all the old swap contents can be discarded, 90 * to allow the swap device to optimize its wear-levelling. 91 */ 92static int discard_swap(struct swap_info_struct *si) 93{ 94 struct swap_extent *se; 95 int err = 0; 96 97 list_for_each_entry(se, &si->extent_list, list) { 98 sector_t start_block = se->start_block << (PAGE_SHIFT - 9); 99 pgoff_t nr_blocks = se->nr_pages << (PAGE_SHIFT - 9); 100 101 if (se->start_page == 0) { 102 /* Do not discard the swap header page! */ 103 start_block += 1 << (PAGE_SHIFT - 9); 104 nr_blocks -= 1 << (PAGE_SHIFT - 9); 105 if (!nr_blocks) 106 continue; 107 } 108 109 err = blkdev_issue_discard(si->bdev, start_block, 110 nr_blocks, GFP_KERNEL); 111 if (err) 112 break; 113 114 cond_resched(); 115 } 116 return err; /* That will often be -EOPNOTSUPP */ 117} 118 119/* 120 * swap allocation tell device that a cluster of swap can now be discarded, 121 * to allow the swap device to optimize its wear-levelling. 122 */ 123static void discard_swap_cluster(struct swap_info_struct *si, 124 pgoff_t start_page, pgoff_t nr_pages) 125{ 126 struct swap_extent *se = si->curr_swap_extent; 127 int found_extent = 0; 128 129 while (nr_pages) { 130 struct list_head *lh; 131 132 if (se->start_page <= start_page && 133 start_page < se->start_page + se->nr_pages) { 134 pgoff_t offset = start_page - se->start_page; 135 sector_t start_block = se->start_block + offset; 136 pgoff_t nr_blocks = se->nr_pages - offset; 137 138 if (nr_blocks > nr_pages) 139 nr_blocks = nr_pages; 140 start_page += nr_blocks; 141 nr_pages -= nr_blocks; 142 143 if (!found_extent++) 144 si->curr_swap_extent = se; 145 146 start_block <<= PAGE_SHIFT - 9; 147 nr_blocks <<= PAGE_SHIFT - 9; 148 if (blkdev_issue_discard(si->bdev, start_block, 149 nr_blocks, GFP_NOIO)) 150 break; 151 } 152 153 lh = se->list.next; 154 if (lh == &si->extent_list) 155 lh = lh->next; 156 se = list_entry(lh, struct swap_extent, list); 157 } 158} 159 160static int wait_for_discard(void *word) 161{ 162 schedule(); 163 return 0; 164} 165 166#define SWAPFILE_CLUSTER 256 167#define LATENCY_LIMIT 256 168 169static inline unsigned long scan_swap_map(struct swap_info_struct *si) 170{ 171 unsigned long offset; 172 unsigned long last_in_cluster = 0; 173 int latency_ration = LATENCY_LIMIT; 174 int found_free_cluster = 0; 175 176 /* 177 * We try to cluster swap pages by allocating them sequentially 178 * in swap. Once we've allocated SWAPFILE_CLUSTER pages this 179 * way, however, we resort to first-free allocation, starting 180 * a new cluster. This prevents us from scattering swap pages 181 * all over the entire swap partition, so that we reduce 182 * overall disk seek times between swap pages. -- sct 183 * But we do now try to find an empty cluster. -Andrea 184 */ 185 186 si->flags += SWP_SCANNING; 187 offset = si->cluster_next; 188 189 if (unlikely(!si->cluster_nr--)) { 190 if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) { 191 si->cluster_nr = SWAPFILE_CLUSTER - 1; 192 goto checks; 193 } 194 if (si->flags & SWP_DISCARDABLE) { 195 /* 196 * Start range check on racing allocations, in case 197 * they overlap the cluster we eventually decide on 198 * (we scan without swap_lock to allow preemption). 199 * It's hardly conceivable that cluster_nr could be 200 * wrapped during our scan, but don't depend on it. 201 */ 202 if (si->lowest_alloc) 203 goto checks; 204 si->lowest_alloc = si->max; 205 si->highest_alloc = 0; 206 } 207 spin_unlock(&swap_lock); 208 209 offset = si->lowest_bit; 210 last_in_cluster = offset + SWAPFILE_CLUSTER - 1; 211 212 /* Locate the first empty (unaligned) cluster */ 213 for (; last_in_cluster <= si->highest_bit; offset++) { 214 if (si->swap_map[offset]) 215 last_in_cluster = offset + SWAPFILE_CLUSTER; 216 else if (offset == last_in_cluster) { 217 spin_lock(&swap_lock); 218 offset -= SWAPFILE_CLUSTER - 1; 219 si->cluster_next = offset; 220 si->cluster_nr = SWAPFILE_CLUSTER - 1; 221 found_free_cluster = 1; 222 goto checks; 223 } 224 if (unlikely(--latency_ration < 0)) { 225 cond_resched(); 226 latency_ration = LATENCY_LIMIT; 227 } 228 } 229 230 offset = si->lowest_bit; 231 spin_lock(&swap_lock); 232 si->cluster_nr = SWAPFILE_CLUSTER - 1; 233 si->lowest_alloc = 0; 234 } 235 236checks: 237 if (!(si->flags & SWP_WRITEOK)) 238 goto no_page; 239 if (!si->highest_bit) 240 goto no_page; 241 if (offset > si->highest_bit) 242 offset = si->lowest_bit; 243 if (si->swap_map[offset]) 244 goto scan; 245 246 if (offset == si->lowest_bit) 247 si->lowest_bit++; 248 if (offset == si->highest_bit) 249 si->highest_bit--; 250 si->inuse_pages++; 251 if (si->inuse_pages == si->pages) { 252 si->lowest_bit = si->max; 253 si->highest_bit = 0; 254 } 255 si->swap_map[offset] = 1; 256 si->cluster_next = offset + 1; 257 si->flags -= SWP_SCANNING; 258 259 if (si->lowest_alloc) { 260 /* 261 * Only set when SWP_DISCARDABLE, and there's a scan 262 * for a free cluster in progress or just completed. 263 */ 264 if (found_free_cluster) { 265 /* 266 * To optimize wear-levelling, discard the 267 * old data of the cluster, taking care not to 268 * discard any of its pages that have already 269 * been allocated by racing tasks (offset has 270 * already stepped over any at the beginning). 271 */ 272 if (offset < si->highest_alloc && 273 si->lowest_alloc <= last_in_cluster) 274 last_in_cluster = si->lowest_alloc - 1; 275 si->flags |= SWP_DISCARDING; 276 spin_unlock(&swap_lock); 277 278 if (offset < last_in_cluster) 279 discard_swap_cluster(si, offset, 280 last_in_cluster - offset + 1); 281 282 spin_lock(&swap_lock); 283 si->lowest_alloc = 0; 284 si->flags &= ~SWP_DISCARDING; 285 286 smp_mb(); /* wake_up_bit advises this */ 287 wake_up_bit(&si->flags, ilog2(SWP_DISCARDING)); 288 289 } else if (si->flags & SWP_DISCARDING) { 290 /* 291 * Delay using pages allocated by racing tasks 292 * until the whole discard has been issued. We 293 * could defer that delay until swap_writepage, 294 * but it's easier to keep this self-contained. 295 */ 296 spin_unlock(&swap_lock); 297 wait_on_bit(&si->flags, ilog2(SWP_DISCARDING), 298 wait_for_discard, TASK_UNINTERRUPTIBLE); 299 spin_lock(&swap_lock); 300 } else { 301 /* 302 * Note pages allocated by racing tasks while 303 * scan for a free cluster is in progress, so 304 * that its final discard can exclude them. 305 */ 306 if (offset < si->lowest_alloc) 307 si->lowest_alloc = offset; 308 if (offset > si->highest_alloc) 309 si->highest_alloc = offset; 310 } 311 } 312 return offset; 313 314scan: 315 spin_unlock(&swap_lock); 316 while (++offset <= si->highest_bit) { 317 if (!si->swap_map[offset]) { 318 spin_lock(&swap_lock); 319 goto checks; 320 } 321 if (unlikely(--latency_ration < 0)) { 322 cond_resched(); 323 latency_ration = LATENCY_LIMIT; 324 } 325 } 326 spin_lock(&swap_lock); 327 goto checks; 328 329no_page: 330 si->flags -= SWP_SCANNING; 331 return 0; 332} 333 334swp_entry_t get_swap_page(void) 335{ 336 struct swap_info_struct *si; 337 pgoff_t offset; 338 int type, next; 339 int wrapped = 0; 340 341 spin_lock(&swap_lock); 342 if (nr_swap_pages <= 0) 343 goto noswap; 344 nr_swap_pages--; 345 346 for (type = swap_list.next; type >= 0 && wrapped < 2; type = next) { 347 si = swap_info + type; 348 next = si->next; 349 if (next < 0 || 350 (!wrapped && si->prio != swap_info[next].prio)) { 351 next = swap_list.head; 352 wrapped++; 353 } 354 355 if (!si->highest_bit) 356 continue; 357 if (!(si->flags & SWP_WRITEOK)) 358 continue; 359 360 swap_list.next = next; 361 offset = scan_swap_map(si); 362 if (offset) { 363 spin_unlock(&swap_lock); 364 return swp_entry(type, offset); 365 } 366 next = swap_list.next; 367 } 368 369 nr_swap_pages++; 370noswap: 371 spin_unlock(&swap_lock); 372 return (swp_entry_t) {0}; 373} 374 375swp_entry_t get_swap_page_of_type(int type) 376{ 377 struct swap_info_struct *si; 378 pgoff_t offset; 379 380 spin_lock(&swap_lock); 381 si = swap_info + type; 382 if (si->flags & SWP_WRITEOK) { 383 nr_swap_pages--; 384 offset = scan_swap_map(si); 385 if (offset) { 386 spin_unlock(&swap_lock); 387 return swp_entry(type, offset); 388 } 389 nr_swap_pages++; 390 } 391 spin_unlock(&swap_lock); 392 return (swp_entry_t) {0}; 393} 394 395static struct swap_info_struct * swap_info_get(swp_entry_t entry) 396{ 397 struct swap_info_struct * p; 398 unsigned long offset, type; 399 400 if (!entry.val) 401 goto out; 402 type = swp_type(entry); 403 if (type >= nr_swapfiles) 404 goto bad_nofile; 405 p = & swap_info[type]; 406 if (!(p->flags & SWP_USED)) 407 goto bad_device; 408 offset = swp_offset(entry); 409 if (offset >= p->max) 410 goto bad_offset; 411 if (!p->swap_map[offset]) 412 goto bad_free; 413 spin_lock(&swap_lock); 414 return p; 415 416bad_free: 417 printk(KERN_ERR "swap_free: %s%08lx\n", Unused_offset, entry.val); 418 goto out; 419bad_offset: 420 printk(KERN_ERR "swap_free: %s%08lx\n", Bad_offset, entry.val); 421 goto out; 422bad_device: 423 printk(KERN_ERR "swap_free: %s%08lx\n", Unused_file, entry.val); 424 goto out; 425bad_nofile: 426 printk(KERN_ERR "swap_free: %s%08lx\n", Bad_file, entry.val); 427out: 428 return NULL; 429} 430 431static int swap_entry_free(struct swap_info_struct *p, unsigned long offset) 432{ 433 int count = p->swap_map[offset]; 434 435 if (count < SWAP_MAP_MAX) { 436 count--; 437 p->swap_map[offset] = count; 438 if (!count) { 439 if (offset < p->lowest_bit) 440 p->lowest_bit = offset; 441 if (offset > p->highest_bit) 442 p->highest_bit = offset; 443 if (p->prio > swap_info[swap_list.next].prio) 444 swap_list.next = p - swap_info; 445 nr_swap_pages++; 446 p->inuse_pages--; 447 } 448 } 449 return count; 450} 451 452/* 453 * Caller has made sure that the swapdevice corresponding to entry 454 * is still around or has not been recycled. 455 */ 456void swap_free(swp_entry_t entry) 457{ 458 struct swap_info_struct * p; 459 460 p = swap_info_get(entry); 461 if (p) { 462 swap_entry_free(p, swp_offset(entry)); 463 spin_unlock(&swap_lock); 464 } 465} 466 467/* 468 * How many references to page are currently swapped out? 469 */ 470static inline int page_swapcount(struct page *page) 471{ 472 int count = 0; 473 struct swap_info_struct *p; 474 swp_entry_t entry; 475 476 entry.val = page_private(page); 477 p = swap_info_get(entry); 478 if (p) { 479 /* Subtract the 1 for the swap cache itself */ 480 count = p->swap_map[swp_offset(entry)] - 1; 481 spin_unlock(&swap_lock); 482 } 483 return count; 484} 485 486/* 487 * We can write to an anon page without COW if there are no other references 488 * to it. And as a side-effect, free up its swap: because the old content 489 * on disk will never be read, and seeking back there to write new content 490 * later would only waste time away from clustering. 491 */ 492int reuse_swap_page(struct page *page) 493{ 494 int count; 495 496 VM_BUG_ON(!PageLocked(page)); 497 count = page_mapcount(page); 498 if (count <= 1 && PageSwapCache(page)) { 499 count += page_swapcount(page); 500 if (count == 1 && !PageWriteback(page)) { 501 delete_from_swap_cache(page); 502 SetPageDirty(page); 503 } 504 } 505 return count == 1; 506} 507 508/* 509 * If swap is getting full, or if there are no more mappings of this page, 510 * then try_to_free_swap is called to free its swap space. 511 */ 512int try_to_free_swap(struct page *page) 513{ 514 VM_BUG_ON(!PageLocked(page)); 515 516 if (!PageSwapCache(page)) 517 return 0; 518 if (PageWriteback(page)) 519 return 0; 520 if (page_swapcount(page)) 521 return 0; 522 523 delete_from_swap_cache(page); 524 SetPageDirty(page); 525 return 1; 526} 527 528/* 529 * Free the swap entry like above, but also try to 530 * free the page cache entry if it is the last user. 531 */ 532void free_swap_and_cache(swp_entry_t entry) 533{ 534 struct swap_info_struct * p; 535 struct page *page = NULL; 536 537 if (is_migration_entry(entry)) 538 return; 539 540 p = swap_info_get(entry); 541 if (p) { 542 if (swap_entry_free(p, swp_offset(entry)) == 1) { 543 page = find_get_page(&swapper_space, entry.val); 544 if (page && !trylock_page(page)) { 545 page_cache_release(page); 546 page = NULL; 547 } 548 } 549 spin_unlock(&swap_lock); 550 } 551 if (page) { 552 /* 553 * Not mapped elsewhere, or swap space full? Free it! 554 * Also recheck PageSwapCache now page is locked (above). 555 */ 556 if (PageSwapCache(page) && !PageWriteback(page) && 557 (!page_mapped(page) || vm_swap_full())) { 558 delete_from_swap_cache(page); 559 SetPageDirty(page); 560 } 561 unlock_page(page); 562 page_cache_release(page); 563 } 564} 565 566#ifdef CONFIG_HIBERNATION 567/* 568 * Find the swap type that corresponds to given device (if any). 569 * 570 * @offset - number of the PAGE_SIZE-sized block of the device, starting 571 * from 0, in which the swap header is expected to be located. 572 * 573 * This is needed for the suspend to disk (aka swsusp). 574 */ 575int swap_type_of(dev_t device, sector_t offset, struct block_device **bdev_p) 576{ 577 struct block_device *bdev = NULL; 578 int i; 579 580 if (device) 581 bdev = bdget(device); 582 583 spin_lock(&swap_lock); 584 for (i = 0; i < nr_swapfiles; i++) { 585 struct swap_info_struct *sis = swap_info + i; 586 587 if (!(sis->flags & SWP_WRITEOK)) 588 continue; 589 590 if (!bdev) { 591 if (bdev_p) 592 *bdev_p = sis->bdev; 593 594 spin_unlock(&swap_lock); 595 return i; 596 } 597 if (bdev == sis->bdev) { 598 struct swap_extent *se; 599 600 se = list_entry(sis->extent_list.next, 601 struct swap_extent, list); 602 if (se->start_block == offset) { 603 if (bdev_p) 604 *bdev_p = sis->bdev; 605 606 spin_unlock(&swap_lock); 607 bdput(bdev); 608 return i; 609 } 610 } 611 } 612 spin_unlock(&swap_lock); 613 if (bdev) 614 bdput(bdev); 615 616 return -ENODEV; 617} 618 619/* 620 * Return either the total number of swap pages of given type, or the number 621 * of free pages of that type (depending on @free) 622 * 623 * This is needed for software suspend 624 */ 625unsigned int count_swap_pages(int type, int free) 626{ 627 unsigned int n = 0; 628 629 if (type < nr_swapfiles) { 630 spin_lock(&swap_lock); 631 if (swap_info[type].flags & SWP_WRITEOK) { 632 n = swap_info[type].pages; 633 if (free) 634 n -= swap_info[type].inuse_pages; 635 } 636 spin_unlock(&swap_lock); 637 } 638 return n; 639} 640#endif 641 642/* 643 * No need to decide whether this PTE shares the swap entry with others, 644 * just let do_wp_page work it out if a write is requested later - to 645 * force COW, vm_page_prot omits write permission from any private vma. 646 */ 647static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, 648 unsigned long addr, swp_entry_t entry, struct page *page) 649{ 650 spinlock_t *ptl; 651 pte_t *pte; 652 int ret = 1; 653 654 if (mem_cgroup_charge(page, vma->vm_mm, GFP_KERNEL)) 655 ret = -ENOMEM; 656 657 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 658 if (unlikely(!pte_same(*pte, swp_entry_to_pte(entry)))) { 659 if (ret > 0) 660 mem_cgroup_uncharge_page(page); 661 ret = 0; 662 goto out; 663 } 664 665 inc_mm_counter(vma->vm_mm, anon_rss); 666 get_page(page); 667 set_pte_at(vma->vm_mm, addr, pte, 668 pte_mkold(mk_pte(page, vma->vm_page_prot))); 669 page_add_anon_rmap(page, vma, addr); 670 swap_free(entry); 671 /* 672 * Move the page to the active list so it is not 673 * immediately swapped out again after swapon. 674 */ 675 activate_page(page); 676out: 677 pte_unmap_unlock(pte, ptl); 678 return ret; 679} 680 681static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, 682 unsigned long addr, unsigned long end, 683 swp_entry_t entry, struct page *page) 684{ 685 pte_t swp_pte = swp_entry_to_pte(entry); 686 pte_t *pte; 687 int ret = 0; 688 689 /* 690 * We don't actually need pte lock while scanning for swp_pte: since 691 * we hold page lock and mmap_sem, swp_pte cannot be inserted into the 692 * page table while we're scanning; though it could get zapped, and on 693 * some architectures (e.g. x86_32 with PAE) we might catch a glimpse 694 * of unmatched parts which look like swp_pte, so unuse_pte must 695 * recheck under pte lock. Scanning without pte lock lets it be 696 * preemptible whenever CONFIG_PREEMPT but not CONFIG_HIGHPTE. 697 */ 698 pte = pte_offset_map(pmd, addr); 699 do { 700 /* 701 * swapoff spends a _lot_ of time in this loop! 702 * Test inline before going to call unuse_pte. 703 */ 704 if (unlikely(pte_same(*pte, swp_pte))) { 705 pte_unmap(pte); 706 ret = unuse_pte(vma, pmd, addr, entry, page); 707 if (ret) 708 goto out; 709 pte = pte_offset_map(pmd, addr); 710 } 711 } while (pte++, addr += PAGE_SIZE, addr != end); 712 pte_unmap(pte - 1); 713out: 714 return ret; 715} 716 717static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud, 718 unsigned long addr, unsigned long end, 719 swp_entry_t entry, struct page *page) 720{ 721 pmd_t *pmd; 722 unsigned long next; 723 int ret; 724 725 pmd = pmd_offset(pud, addr); 726 do { 727 next = pmd_addr_end(addr, end); 728 if (pmd_none_or_clear_bad(pmd)) 729 continue; 730 ret = unuse_pte_range(vma, pmd, addr, next, entry, page); 731 if (ret) 732 return ret; 733 } while (pmd++, addr = next, addr != end); 734 return 0; 735} 736 737static inline int unuse_pud_range(struct vm_area_struct *vma, pgd_t *pgd, 738 unsigned long addr, unsigned long end, 739 swp_entry_t entry, struct page *page) 740{ 741 pud_t *pud; 742 unsigned long next; 743 int ret; 744 745 pud = pud_offset(pgd, addr); 746 do { 747 next = pud_addr_end(addr, end); 748 if (pud_none_or_clear_bad(pud)) 749 continue; 750 ret = unuse_pmd_range(vma, pud, addr, next, entry, page); 751 if (ret) 752 return ret; 753 } while (pud++, addr = next, addr != end); 754 return 0; 755} 756 757static int unuse_vma(struct vm_area_struct *vma, 758 swp_entry_t entry, struct page *page) 759{ 760 pgd_t *pgd; 761 unsigned long addr, end, next; 762 int ret; 763 764 if (page->mapping) { 765 addr = page_address_in_vma(page, vma); 766 if (addr == -EFAULT) 767 return 0; 768 else 769 end = addr + PAGE_SIZE; 770 } else { 771 addr = vma->vm_start; 772 end = vma->vm_end; 773 } 774 775 pgd = pgd_offset(vma->vm_mm, addr); 776 do { 777 next = pgd_addr_end(addr, end); 778 if (pgd_none_or_clear_bad(pgd)) 779 continue; 780 ret = unuse_pud_range(vma, pgd, addr, next, entry, page); 781 if (ret) 782 return ret; 783 } while (pgd++, addr = next, addr != end); 784 return 0; 785} 786 787static int unuse_mm(struct mm_struct *mm, 788 swp_entry_t entry, struct page *page) 789{ 790 struct vm_area_struct *vma; 791 int ret = 0; 792 793 if (!down_read_trylock(&mm->mmap_sem)) { 794 /* 795 * Activate page so shrink_inactive_list is unlikely to unmap 796 * its ptes while lock is dropped, so swapoff can make progress. 797 */ 798 activate_page(page); 799 unlock_page(page); 800 down_read(&mm->mmap_sem); 801 lock_page(page); 802 } 803 for (vma = mm->mmap; vma; vma = vma->vm_next) { 804 if (vma->anon_vma && (ret = unuse_vma(vma, entry, page))) 805 break; 806 } 807 up_read(&mm->mmap_sem); 808 return (ret < 0)? ret: 0; 809} 810 811/* 812 * Scan swap_map from current position to next entry still in use. 813 * Recycle to start on reaching the end, returning 0 when empty. 814 */ 815static unsigned int find_next_to_unuse(struct swap_info_struct *si, 816 unsigned int prev) 817{ 818 unsigned int max = si->max; 819 unsigned int i = prev; 820 int count; 821 822 /* 823 * No need for swap_lock here: we're just looking 824 * for whether an entry is in use, not modifying it; false 825 * hits are okay, and sys_swapoff() has already prevented new 826 * allocations from this area (while holding swap_lock). 827 */ 828 for (;;) { 829 if (++i >= max) { 830 if (!prev) { 831 i = 0; 832 break; 833 } 834 /* 835 * No entries in use at top of swap_map, 836 * loop back to start and recheck there. 837 */ 838 max = prev + 1; 839 prev = 0; 840 i = 1; 841 } 842 count = si->swap_map[i]; 843 if (count && count != SWAP_MAP_BAD) 844 break; 845 } 846 return i; 847} 848 849/* 850 * We completely avoid races by reading each swap page in advance, 851 * and then search for the process using it. All the necessary 852 * page table adjustments can then be made atomically. 853 */ 854static int try_to_unuse(unsigned int type) 855{ 856 struct swap_info_struct * si = &swap_info[type]; 857 struct mm_struct *start_mm; 858 unsigned short *swap_map; 859 unsigned short swcount; 860 struct page *page; 861 swp_entry_t entry; 862 unsigned int i = 0; 863 int retval = 0; 864 int reset_overflow = 0; 865 int shmem; 866 867 /* 868 * When searching mms for an entry, a good strategy is to 869 * start at the first mm we freed the previous entry from 870 * (though actually we don't notice whether we or coincidence 871 * freed the entry). Initialize this start_mm with a hold. 872 * 873 * A simpler strategy would be to start at the last mm we 874 * freed the previous entry from; but that would take less 875 * advantage of mmlist ordering, which clusters forked mms 876 * together, child after parent. If we race with dup_mmap(), we 877 * prefer to resolve parent before child, lest we miss entries 878 * duplicated after we scanned child: using last mm would invert 879 * that. Though it's only a serious concern when an overflowed 880 * swap count is reset from SWAP_MAP_MAX, preventing a rescan. 881 */ 882 start_mm = &init_mm; 883 atomic_inc(&init_mm.mm_users); 884 885 /* 886 * Keep on scanning until all entries have gone. Usually, 887 * one pass through swap_map is enough, but not necessarily: 888 * there are races when an instance of an entry might be missed. 889 */ 890 while ((i = find_next_to_unuse(si, i)) != 0) { 891 if (signal_pending(current)) { 892 retval = -EINTR; 893 break; 894 } 895 896 /* 897 * Get a page for the entry, using the existing swap 898 * cache page if there is one. Otherwise, get a clean 899 * page and read the swap into it. 900 */ 901 swap_map = &si->swap_map[i]; 902 entry = swp_entry(type, i); 903 page = read_swap_cache_async(entry, 904 GFP_HIGHUSER_MOVABLE, NULL, 0); 905 if (!page) { 906 /* 907 * Either swap_duplicate() failed because entry 908 * has been freed independently, and will not be 909 * reused since sys_swapoff() already disabled 910 * allocation from here, or alloc_page() failed. 911 */ 912 if (!*swap_map) 913 continue; 914 retval = -ENOMEM; 915 break; 916 } 917 918 /* 919 * Don't hold on to start_mm if it looks like exiting. 920 */ 921 if (atomic_read(&start_mm->mm_users) == 1) { 922 mmput(start_mm); 923 start_mm = &init_mm; 924 atomic_inc(&init_mm.mm_users); 925 } 926 927 /* 928 * Wait for and lock page. When do_swap_page races with 929 * try_to_unuse, do_swap_page can handle the fault much 930 * faster than try_to_unuse can locate the entry. This 931 * apparently redundant "wait_on_page_locked" lets try_to_unuse 932 * defer to do_swap_page in such a case - in some tests, 933 * do_swap_page and try_to_unuse repeatedly compete. 934 */ 935 wait_on_page_locked(page); 936 wait_on_page_writeback(page); 937 lock_page(page); 938 wait_on_page_writeback(page); 939 940 /* 941 * Remove all references to entry. 942 * Whenever we reach init_mm, there's no address space 943 * to search, but use it as a reminder to search shmem. 944 */ 945 shmem = 0; 946 swcount = *swap_map; 947 if (swcount > 1) { 948 if (start_mm == &init_mm) 949 shmem = shmem_unuse(entry, page); 950 else 951 retval = unuse_mm(start_mm, entry, page); 952 } 953 if (*swap_map > 1) { 954 int set_start_mm = (*swap_map >= swcount); 955 struct list_head *p = &start_mm->mmlist; 956 struct mm_struct *new_start_mm = start_mm; 957 struct mm_struct *prev_mm = start_mm; 958 struct mm_struct *mm; 959 960 atomic_inc(&new_start_mm->mm_users); 961 atomic_inc(&prev_mm->mm_users); 962 spin_lock(&mmlist_lock); 963 while (*swap_map > 1 && !retval && !shmem && 964 (p = p->next) != &start_mm->mmlist) { 965 mm = list_entry(p, struct mm_struct, mmlist); 966 if (!atomic_inc_not_zero(&mm->mm_users)) 967 continue; 968 spin_unlock(&mmlist_lock); 969 mmput(prev_mm); 970 prev_mm = mm; 971 972 cond_resched(); 973 974 swcount = *swap_map; 975 if (swcount <= 1) 976 ; 977 else if (mm == &init_mm) { 978 set_start_mm = 1; 979 shmem = shmem_unuse(entry, page); 980 } else 981 retval = unuse_mm(mm, entry, page); 982 if (set_start_mm && *swap_map < swcount) { 983 mmput(new_start_mm); 984 atomic_inc(&mm->mm_users); 985 new_start_mm = mm; 986 set_start_mm = 0; 987 } 988 spin_lock(&mmlist_lock); 989 } 990 spin_unlock(&mmlist_lock); 991 mmput(prev_mm); 992 mmput(start_mm); 993 start_mm = new_start_mm; 994 } 995 if (shmem) { 996 /* page has already been unlocked and released */ 997 if (shmem > 0) 998 continue; 999 retval = shmem; 1000 break; 1001 } 1002 if (retval) { 1003 unlock_page(page); 1004 page_cache_release(page); 1005 break; 1006 } 1007 1008 /* 1009 * How could swap count reach 0x7fff when the maximum 1010 * pid is 0x7fff, and there's no way to repeat a swap 1011 * page within an mm (except in shmem, where it's the 1012 * shared object which takes the reference count)? 1013 * We believe SWAP_MAP_MAX cannot occur in Linux 2.4. 1014 * 1015 * If that's wrong, then we should worry more about 1016 * exit_mmap() and do_munmap() cases described above: 1017 * we might be resetting SWAP_MAP_MAX too early here. 1018 * We know "Undead"s can happen, they're okay, so don't 1019 * report them; but do report if we reset SWAP_MAP_MAX. 1020 */ 1021 if (*swap_map == SWAP_MAP_MAX) { 1022 spin_lock(&swap_lock); 1023 *swap_map = 1; 1024 spin_unlock(&swap_lock); 1025 reset_overflow = 1; 1026 } 1027 1028 /* 1029 * If a reference remains (rare), we would like to leave 1030 * the page in the swap cache; but try_to_unmap could 1031 * then re-duplicate the entry once we drop page lock, 1032 * so we might loop indefinitely; also, that page could 1033 * not be swapped out to other storage meanwhile. So: 1034 * delete from cache even if there's another reference, 1035 * after ensuring that the data has been saved to disk - 1036 * since if the reference remains (rarer), it will be 1037 * read from disk into another page. Splitting into two 1038 * pages would be incorrect if swap supported "shared 1039 * private" pages, but they are handled by tmpfs files. 1040 */ 1041 if ((*swap_map > 1) && PageDirty(page) && PageSwapCache(page)) { 1042 struct writeback_control wbc = { 1043 .sync_mode = WB_SYNC_NONE, 1044 }; 1045 1046 swap_writepage(page, &wbc); 1047 lock_page(page); 1048 wait_on_page_writeback(page); 1049 } 1050 1051 /* 1052 * It is conceivable that a racing task removed this page from 1053 * swap cache just before we acquired the page lock at the top, 1054 * or while we dropped it in unuse_mm(). The page might even 1055 * be back in swap cache on another swap area: that we must not 1056 * delete, since it may not have been written out to swap yet. 1057 */ 1058 if (PageSwapCache(page) && 1059 likely(page_private(page) == entry.val)) 1060 delete_from_swap_cache(page); 1061 1062 /* 1063 * So we could skip searching mms once swap count went 1064 * to 1, we did not mark any present ptes as dirty: must 1065 * mark page dirty so shrink_page_list will preserve it. 1066 */ 1067 SetPageDirty(page); 1068 unlock_page(page); 1069 page_cache_release(page); 1070 1071 /* 1072 * Make sure that we aren't completely killing 1073 * interactive performance. 1074 */ 1075 cond_resched(); 1076 } 1077 1078 mmput(start_mm); 1079 if (reset_overflow) { 1080 printk(KERN_WARNING "swapoff: cleared swap entry overflow\n"); 1081 swap_overflow = 0; 1082 } 1083 return retval; 1084} 1085 1086/* 1087 * After a successful try_to_unuse, if no swap is now in use, we know 1088 * we can empty the mmlist. swap_lock must be held on entry and exit. 1089 * Note that mmlist_lock nests inside swap_lock, and an mm must be 1090 * added to the mmlist just after page_duplicate - before would be racy. 1091 */ 1092static void drain_mmlist(void) 1093{ 1094 struct list_head *p, *next; 1095 unsigned int i; 1096 1097 for (i = 0; i < nr_swapfiles; i++) 1098 if (swap_info[i].inuse_pages) 1099 return; 1100 spin_lock(&mmlist_lock); 1101 list_for_each_safe(p, next, &init_mm.mmlist) 1102 list_del_init(p); 1103 spin_unlock(&mmlist_lock); 1104} 1105 1106/* 1107 * Use this swapdev's extent info to locate the (PAGE_SIZE) block which 1108 * corresponds to page offset `offset'. 1109 */ 1110sector_t map_swap_page(struct swap_info_struct *sis, pgoff_t offset) 1111{ 1112 struct swap_extent *se = sis->curr_swap_extent; 1113 struct swap_extent *start_se = se; 1114 1115 for ( ; ; ) { 1116 struct list_head *lh; 1117 1118 if (se->start_page <= offset && 1119 offset < (se->start_page + se->nr_pages)) { 1120 return se->start_block + (offset - se->start_page); 1121 } 1122 lh = se->list.next; 1123 if (lh == &sis->extent_list) 1124 lh = lh->next; 1125 se = list_entry(lh, struct swap_extent, list); 1126 sis->curr_swap_extent = se; 1127 BUG_ON(se == start_se); /* It *must* be present */ 1128 } 1129} 1130 1131#ifdef CONFIG_HIBERNATION 1132/* 1133 * Get the (PAGE_SIZE) block corresponding to given offset on the swapdev 1134 * corresponding to given index in swap_info (swap type). 1135 */ 1136sector_t swapdev_block(int swap_type, pgoff_t offset) 1137{ 1138 struct swap_info_struct *sis; 1139 1140 if (swap_type >= nr_swapfiles) 1141 return 0; 1142 1143 sis = swap_info + swap_type; 1144 return (sis->flags & SWP_WRITEOK) ? map_swap_page(sis, offset) : 0; 1145} 1146#endif /* CONFIG_HIBERNATION */ 1147 1148/* 1149 * Free all of a swapdev's extent information 1150 */ 1151static void destroy_swap_extents(struct swap_info_struct *sis) 1152{ 1153 while (!list_empty(&sis->extent_list)) { 1154 struct swap_extent *se; 1155 1156 se = list_entry(sis->extent_list.next, 1157 struct swap_extent, list); 1158 list_del(&se->list); 1159 kfree(se); 1160 } 1161} 1162 1163/* 1164 * Add a block range (and the corresponding page range) into this swapdev's 1165 * extent list. The extent list is kept sorted in page order. 1166 * 1167 * This function rather assumes that it is called in ascending page order. 1168 */ 1169static int 1170add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, 1171 unsigned long nr_pages, sector_t start_block) 1172{ 1173 struct swap_extent *se; 1174 struct swap_extent *new_se; 1175 struct list_head *lh; 1176 1177 lh = sis->extent_list.prev; /* The highest page extent */ 1178 if (lh != &sis->extent_list) { 1179 se = list_entry(lh, struct swap_extent, list); 1180 BUG_ON(se->start_page + se->nr_pages != start_page); 1181 if (se->start_block + se->nr_pages == start_block) { 1182 /* Merge it */ 1183 se->nr_pages += nr_pages; 1184 return 0; 1185 } 1186 } 1187 1188 /* 1189 * No merge. Insert a new extent, preserving ordering. 1190 */ 1191 new_se = kmalloc(sizeof(*se), GFP_KERNEL); 1192 if (new_se == NULL) 1193 return -ENOMEM; 1194 new_se->start_page = start_page; 1195 new_se->nr_pages = nr_pages; 1196 new_se->start_block = start_block; 1197 1198 list_add_tail(&new_se->list, &sis->extent_list); 1199 return 1; 1200} 1201 1202/* 1203 * A `swap extent' is a simple thing which maps a contiguous range of pages 1204 * onto a contiguous range of disk blocks. An ordered list of swap extents 1205 * is built at swapon time and is then used at swap_writepage/swap_readpage 1206 * time for locating where on disk a page belongs. 1207 * 1208 * If the swapfile is an S_ISBLK block device, a single extent is installed. 1209 * This is done so that the main operating code can treat S_ISBLK and S_ISREG 1210 * swap files identically. 1211 * 1212 * Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap 1213 * extent list operates in PAGE_SIZE disk blocks. Both S_ISREG and S_ISBLK 1214 * swapfiles are handled *identically* after swapon time. 1215 * 1216 * For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks 1217 * and will parse them into an ordered extent list, in PAGE_SIZE chunks. If 1218 * some stray blocks are found which do not fall within the PAGE_SIZE alignment 1219 * requirements, they are simply tossed out - we will never use those blocks 1220 * for swapping. 1221 * 1222 * For S_ISREG swapfiles we set S_SWAPFILE across the life of the swapon. This 1223 * prevents root from shooting her foot off by ftruncating an in-use swapfile, 1224 * which will scribble on the fs. 1225 * 1226 * The amount of disk space which a single swap extent represents varies. 1227 * Typically it is in the 1-4 megabyte range. So we can have hundreds of 1228 * extents in the list. To avoid much list walking, we cache the previous 1229 * search location in `curr_swap_extent', and start new searches from there. 1230 * This is extremely effective. The average number of iterations in 1231 * map_swap_page() has been measured at about 0.3 per page. - akpm. 1232 */ 1233static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span) 1234{ 1235 struct inode *inode; 1236 unsigned blocks_per_page; 1237 unsigned long page_no; 1238 unsigned blkbits; 1239 sector_t probe_block; 1240 sector_t last_block; 1241 sector_t lowest_block = -1; 1242 sector_t highest_block = 0; 1243 int nr_extents = 0; 1244 int ret; 1245 1246 inode = sis->swap_file->f_mapping->host; 1247 if (S_ISBLK(inode->i_mode)) { 1248 ret = add_swap_extent(sis, 0, sis->max, 0); 1249 *span = sis->pages; 1250 goto done; 1251 } 1252 1253 blkbits = inode->i_blkbits; 1254 blocks_per_page = PAGE_SIZE >> blkbits; 1255 1256 /* 1257 * Map all the blocks into the extent list. This code doesn't try 1258 * to be very smart. 1259 */ 1260 probe_block = 0; 1261 page_no = 0; 1262 last_block = i_size_read(inode) >> blkbits; 1263 while ((probe_block + blocks_per_page) <= last_block && 1264 page_no < sis->max) { 1265 unsigned block_in_page; 1266 sector_t first_block; 1267 1268 first_block = bmap(inode, probe_block); 1269 if (first_block == 0) 1270 goto bad_bmap; 1271 1272 /* 1273 * It must be PAGE_SIZE aligned on-disk 1274 */ 1275 if (first_block & (blocks_per_page - 1)) { 1276 probe_block++; 1277 goto reprobe; 1278 } 1279 1280 for (block_in_page = 1; block_in_page < blocks_per_page; 1281 block_in_page++) { 1282 sector_t block; 1283 1284 block = bmap(inode, probe_block + block_in_page); 1285 if (block == 0) 1286 goto bad_bmap; 1287 if (block != first_block + block_in_page) { 1288 /* Discontiguity */ 1289 probe_block++; 1290 goto reprobe; 1291 } 1292 } 1293 1294 first_block >>= (PAGE_SHIFT - blkbits); 1295 if (page_no) { /* exclude the header page */ 1296 if (first_block < lowest_block) 1297 lowest_block = first_block; 1298 if (first_block > highest_block) 1299 highest_block = first_block; 1300 } 1301 1302 /* 1303 * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks 1304 */ 1305 ret = add_swap_extent(sis, page_no, 1, first_block); 1306 if (ret < 0) 1307 goto out; 1308 nr_extents += ret; 1309 page_no++; 1310 probe_block += blocks_per_page; 1311reprobe: 1312 continue; 1313 } 1314 ret = nr_extents; 1315 *span = 1 + highest_block - lowest_block; 1316 if (page_no == 0) 1317 page_no = 1; /* force Empty message */ 1318 sis->max = page_no; 1319 sis->pages = page_no - 1; 1320 sis->highest_bit = page_no - 1; 1321done: 1322 sis->curr_swap_extent = list_entry(sis->extent_list.prev, 1323 struct swap_extent, list); 1324 goto out; 1325bad_bmap: 1326 printk(KERN_ERR "swapon: swapfile has holes\n"); 1327 ret = -EINVAL; 1328out: 1329 return ret; 1330} 1331 1332#if 0 /* We don't need this yet */ 1333#include <linux/backing-dev.h> 1334int page_queue_congested(struct page *page) 1335{ 1336 struct backing_dev_info *bdi; 1337 1338 VM_BUG_ON(!PageLocked(page)); /* It pins the swap_info_struct */ 1339 1340 if (PageSwapCache(page)) { 1341 swp_entry_t entry = { .val = page_private(page) }; 1342 struct swap_info_struct *sis; 1343 1344 sis = get_swap_info_struct(swp_type(entry)); 1345 bdi = sis->bdev->bd_inode->i_mapping->backing_dev_info; 1346 } else 1347 bdi = page->mapping->backing_dev_info; 1348 return bdi_write_congested(bdi); 1349} 1350#endif 1351 1352asmlinkage long sys_swapoff(const char __user * specialfile) 1353{ 1354 struct swap_info_struct * p = NULL; 1355 unsigned short *swap_map; 1356 struct file *swap_file, *victim; 1357 struct address_space *mapping; 1358 struct inode *inode; 1359 char * pathname; 1360 int i, type, prev; 1361 int err; 1362 1363 if (!capable(CAP_SYS_ADMIN)) 1364 return -EPERM; 1365 1366 pathname = getname(specialfile); 1367 err = PTR_ERR(pathname); 1368 if (IS_ERR(pathname)) 1369 goto out; 1370 1371 victim = filp_open(pathname, O_RDWR|O_LARGEFILE, 0); 1372 putname(pathname); 1373 err = PTR_ERR(victim); 1374 if (IS_ERR(victim)) 1375 goto out; 1376 1377 mapping = victim->f_mapping; 1378 prev = -1; 1379 spin_lock(&swap_lock); 1380 for (type = swap_list.head; type >= 0; type = swap_info[type].next) { 1381 p = swap_info + type; 1382 if (p->flags & SWP_WRITEOK) { 1383 if (p->swap_file->f_mapping == mapping) 1384 break; 1385 } 1386 prev = type; 1387 } 1388 if (type < 0) { 1389 err = -EINVAL; 1390 spin_unlock(&swap_lock); 1391 goto out_dput; 1392 } 1393 if (!security_vm_enough_memory(p->pages)) 1394 vm_unacct_memory(p->pages); 1395 else { 1396 err = -ENOMEM; 1397 spin_unlock(&swap_lock); 1398 goto out_dput; 1399 } 1400 if (prev < 0) { 1401 swap_list.head = p->next; 1402 } else { 1403 swap_info[prev].next = p->next; 1404 } 1405 if (type == swap_list.next) { 1406 /* just pick something that's safe... */ 1407 swap_list.next = swap_list.head; 1408 } 1409 if (p->prio < 0) { 1410 for (i = p->next; i >= 0; i = swap_info[i].next) 1411 swap_info[i].prio = p->prio--; 1412 least_priority++; 1413 } 1414 nr_swap_pages -= p->pages; 1415 total_swap_pages -= p->pages; 1416 p->flags &= ~SWP_WRITEOK; 1417 spin_unlock(&swap_lock); 1418 1419 current->flags |= PF_SWAPOFF; 1420 err = try_to_unuse(type); 1421 current->flags &= ~PF_SWAPOFF; 1422 1423 if (err) { 1424 /* re-insert swap space back into swap_list */ 1425 spin_lock(&swap_lock); 1426 if (p->prio < 0) 1427 p->prio = --least_priority; 1428 prev = -1; 1429 for (i = swap_list.head; i >= 0; i = swap_info[i].next) { 1430 if (p->prio >= swap_info[i].prio) 1431 break; 1432 prev = i; 1433 } 1434 p->next = i; 1435 if (prev < 0) 1436 swap_list.head = swap_list.next = p - swap_info; 1437 else 1438 swap_info[prev].next = p - swap_info; 1439 nr_swap_pages += p->pages; 1440 total_swap_pages += p->pages; 1441 p->flags |= SWP_WRITEOK; 1442 spin_unlock(&swap_lock); 1443 goto out_dput; 1444 } 1445 1446 /* wait for any unplug function to finish */ 1447 down_write(&swap_unplug_sem); 1448 up_write(&swap_unplug_sem); 1449 1450 destroy_swap_extents(p); 1451 mutex_lock(&swapon_mutex); 1452 spin_lock(&swap_lock); 1453 drain_mmlist(); 1454 1455 /* wait for anyone still in scan_swap_map */ 1456 p->highest_bit = 0; /* cuts scans short */ 1457 while (p->flags >= SWP_SCANNING) { 1458 spin_unlock(&swap_lock); 1459 schedule_timeout_uninterruptible(1); 1460 spin_lock(&swap_lock); 1461 } 1462 1463 swap_file = p->swap_file; 1464 p->swap_file = NULL; 1465 p->max = 0; 1466 swap_map = p->swap_map; 1467 p->swap_map = NULL; 1468 p->flags = 0; 1469 spin_unlock(&swap_lock); 1470 mutex_unlock(&swapon_mutex); 1471 vfree(swap_map); 1472 inode = mapping->host; 1473 if (S_ISBLK(inode->i_mode)) { 1474 struct block_device *bdev = I_BDEV(inode); 1475 set_blocksize(bdev, p->old_block_size); 1476 bd_release(bdev); 1477 } else { 1478 mutex_lock(&inode->i_mutex); 1479 inode->i_flags &= ~S_SWAPFILE; 1480 mutex_unlock(&inode->i_mutex); 1481 } 1482 filp_close(swap_file, NULL); 1483 err = 0; 1484 1485out_dput: 1486 filp_close(victim, NULL); 1487out: 1488 return err; 1489} 1490 1491#ifdef CONFIG_PROC_FS 1492/* iterator */ 1493static void *swap_start(struct seq_file *swap, loff_t *pos) 1494{ 1495 struct swap_info_struct *ptr = swap_info; 1496 int i; 1497 loff_t l = *pos; 1498 1499 mutex_lock(&swapon_mutex); 1500 1501 if (!l) 1502 return SEQ_START_TOKEN; 1503 1504 for (i = 0; i < nr_swapfiles; i++, ptr++) { 1505 if (!(ptr->flags & SWP_USED) || !ptr->swap_map) 1506 continue; 1507 if (!--l) 1508 return ptr; 1509 } 1510 1511 return NULL; 1512} 1513 1514static void *swap_next(struct seq_file *swap, void *v, loff_t *pos) 1515{ 1516 struct swap_info_struct *ptr; 1517 struct swap_info_struct *endptr = swap_info + nr_swapfiles; 1518 1519 if (v == SEQ_START_TOKEN) 1520 ptr = swap_info; 1521 else { 1522 ptr = v; 1523 ptr++; 1524 } 1525 1526 for (; ptr < endptr; ptr++) { 1527 if (!(ptr->flags & SWP_USED) || !ptr->swap_map) 1528 continue; 1529 ++*pos; 1530 return ptr; 1531 } 1532 1533 return NULL; 1534} 1535 1536static void swap_stop(struct seq_file *swap, void *v) 1537{ 1538 mutex_unlock(&swapon_mutex); 1539} 1540 1541static int swap_show(struct seq_file *swap, void *v) 1542{ 1543 struct swap_info_struct *ptr = v; 1544 struct file *file; 1545 int len; 1546 1547 if (ptr == SEQ_START_TOKEN) { 1548 seq_puts(swap,"Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n"); 1549 return 0; 1550 } 1551 1552 file = ptr->swap_file; 1553 len = seq_path(swap, &file->f_path, " \t\n\\"); 1554 seq_printf(swap, "%*s%s\t%u\t%u\t%d\n", 1555 len < 40 ? 40 - len : 1, " ", 1556 S_ISBLK(file->f_path.dentry->d_inode->i_mode) ? 1557 "partition" : "file\t", 1558 ptr->pages << (PAGE_SHIFT - 10), 1559 ptr->inuse_pages << (PAGE_SHIFT - 10), 1560 ptr->prio); 1561 return 0; 1562} 1563 1564static const struct seq_operations swaps_op = { 1565 .start = swap_start, 1566 .next = swap_next, 1567 .stop = swap_stop, 1568 .show = swap_show 1569}; 1570 1571static int swaps_open(struct inode *inode, struct file *file) 1572{ 1573 return seq_open(file, &swaps_op); 1574} 1575 1576static const struct file_operations proc_swaps_operations = { 1577 .open = swaps_open, 1578 .read = seq_read, 1579 .llseek = seq_lseek, 1580 .release = seq_release, 1581}; 1582 1583static int __init procswaps_init(void) 1584{ 1585 proc_create("swaps", 0, NULL, &proc_swaps_operations); 1586 return 0; 1587} 1588__initcall(procswaps_init); 1589#endif /* CONFIG_PROC_FS */ 1590 1591#ifdef MAX_SWAPFILES_CHECK 1592static int __init max_swapfiles_check(void) 1593{ 1594 MAX_SWAPFILES_CHECK(); 1595 return 0; 1596} 1597late_initcall(max_swapfiles_check); 1598#endif 1599 1600/* 1601 * Written 01/25/92 by Simmule Turner, heavily changed by Linus. 1602 * 1603 * The swapon system call 1604 */ 1605asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags) 1606{ 1607 struct swap_info_struct * p; 1608 char *name = NULL; 1609 struct block_device *bdev = NULL; 1610 struct file *swap_file = NULL; 1611 struct address_space *mapping; 1612 unsigned int type; 1613 int i, prev; 1614 int error; 1615 union swap_header *swap_header = NULL; 1616 unsigned int nr_good_pages = 0; 1617 int nr_extents = 0; 1618 sector_t span; 1619 unsigned long maxpages = 1; 1620 unsigned long swapfilepages; 1621 unsigned short *swap_map = NULL; 1622 struct page *page = NULL; 1623 struct inode *inode = NULL; 1624 int did_down = 0; 1625 1626 if (!capable(CAP_SYS_ADMIN)) 1627 return -EPERM; 1628 spin_lock(&swap_lock); 1629 p = swap_info; 1630 for (type = 0 ; type < nr_swapfiles ; type++,p++) 1631 if (!(p->flags & SWP_USED)) 1632 break; 1633 error = -EPERM; 1634 if (type >= MAX_SWAPFILES) { 1635 spin_unlock(&swap_lock); 1636 goto out; 1637 } 1638 if (type >= nr_swapfiles) 1639 nr_swapfiles = type+1; 1640 memset(p, 0, sizeof(*p)); 1641 INIT_LIST_HEAD(&p->extent_list); 1642 p->flags = SWP_USED; 1643 p->next = -1; 1644 spin_unlock(&swap_lock); 1645 name = getname(specialfile); 1646 error = PTR_ERR(name); 1647 if (IS_ERR(name)) { 1648 name = NULL; 1649 goto bad_swap_2; 1650 } 1651 swap_file = filp_open(name, O_RDWR|O_LARGEFILE, 0); 1652 error = PTR_ERR(swap_file); 1653 if (IS_ERR(swap_file)) { 1654 swap_file = NULL; 1655 goto bad_swap_2; 1656 } 1657 1658 p->swap_file = swap_file; 1659 mapping = swap_file->f_mapping; 1660 inode = mapping->host; 1661 1662 error = -EBUSY; 1663 for (i = 0; i < nr_swapfiles; i++) { 1664 struct swap_info_struct *q = &swap_info[i]; 1665 1666 if (i == type || !q->swap_file) 1667 continue; 1668 if (mapping == q->swap_file->f_mapping) 1669 goto bad_swap; 1670 } 1671 1672 error = -EINVAL; 1673 if (S_ISBLK(inode->i_mode)) { 1674 bdev = I_BDEV(inode); 1675 error = bd_claim(bdev, sys_swapon); 1676 if (error < 0) { 1677 bdev = NULL; 1678 error = -EINVAL; 1679 goto bad_swap; 1680 } 1681 p->old_block_size = block_size(bdev); 1682 error = set_blocksize(bdev, PAGE_SIZE); 1683 if (error < 0) 1684 goto bad_swap; 1685 p->bdev = bdev; 1686 } else if (S_ISREG(inode->i_mode)) { 1687 p->bdev = inode->i_sb->s_bdev; 1688 mutex_lock(&inode->i_mutex); 1689 did_down = 1; 1690 if (IS_SWAPFILE(inode)) { 1691 error = -EBUSY; 1692 goto bad_swap; 1693 } 1694 } else { 1695 goto bad_swap; 1696 } 1697 1698 swapfilepages = i_size_read(inode) >> PAGE_SHIFT; 1699 1700 /* 1701 * Read the swap header. 1702 */ 1703 if (!mapping->a_ops->readpage) { 1704 error = -EINVAL; 1705 goto bad_swap; 1706 } 1707 page = read_mapping_page(mapping, 0, swap_file); 1708 if (IS_ERR(page)) { 1709 error = PTR_ERR(page); 1710 goto bad_swap; 1711 } 1712 swap_header = kmap(page); 1713 1714 if (memcmp("SWAPSPACE2", swap_header->magic.magic, 10)) { 1715 printk(KERN_ERR "Unable to find swap-space signature\n"); 1716 error = -EINVAL; 1717 goto bad_swap; 1718 } 1719 1720 /* swap partition endianess hack... */ 1721 if (swab32(swap_header->info.version) == 1) { 1722 swab32s(&swap_header->info.version); 1723 swab32s(&swap_header->info.last_page); 1724 swab32s(&swap_header->info.nr_badpages); 1725 for (i = 0; i < swap_header->info.nr_badpages; i++) 1726 swab32s(&swap_header->info.badpages[i]); 1727 } 1728 /* Check the swap header's sub-version */ 1729 if (swap_header->info.version != 1) { 1730 printk(KERN_WARNING 1731 "Unable to handle swap header version %d\n", 1732 swap_header->info.version); 1733 error = -EINVAL; 1734 goto bad_swap; 1735 } 1736 1737 p->lowest_bit = 1; 1738 p->cluster_next = 1; 1739 1740 /* 1741 * Find out how many pages are allowed for a single swap 1742 * device. There are two limiting factors: 1) the number of 1743 * bits for the swap offset in the swp_entry_t type and 1744 * 2) the number of bits in the a swap pte as defined by 1745 * the different architectures. In order to find the 1746 * largest possible bit mask a swap entry with swap type 0 1747 * and swap offset ~0UL is created, encoded to a swap pte, 1748 * decoded to a swp_entry_t again and finally the swap 1749 * offset is extracted. This will mask all the bits from 1750 * the initial ~0UL mask that can't be encoded in either 1751 * the swp_entry_t or the architecture definition of a 1752 * swap pte. 1753 */ 1754 maxpages = swp_offset(pte_to_swp_entry( 1755 swp_entry_to_pte(swp_entry(0, ~0UL)))) - 1; 1756 if (maxpages > swap_header->info.last_page) 1757 maxpages = swap_header->info.last_page; 1758 p->highest_bit = maxpages - 1; 1759 1760 error = -EINVAL; 1761 if (!maxpages) 1762 goto bad_swap; 1763 if (swapfilepages && maxpages > swapfilepages) { 1764 printk(KERN_WARNING 1765 "Swap area shorter than signature indicates\n"); 1766 goto bad_swap; 1767 } 1768 if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode)) 1769 goto bad_swap; 1770 if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES) 1771 goto bad_swap; 1772 1773 /* OK, set up the swap map and apply the bad block list */ 1774 swap_map = vmalloc(maxpages * sizeof(short)); 1775 if (!swap_map) { 1776 error = -ENOMEM; 1777 goto bad_swap; 1778 } 1779 1780 memset(swap_map, 0, maxpages * sizeof(short)); 1781 for (i = 0; i < swap_header->info.nr_badpages; i++) { 1782 int page_nr = swap_header->info.badpages[i]; 1783 if (page_nr <= 0 || page_nr >= swap_header->info.last_page) { 1784 error = -EINVAL; 1785 goto bad_swap; 1786 } 1787 swap_map[page_nr] = SWAP_MAP_BAD; 1788 } 1789 nr_good_pages = swap_header->info.last_page - 1790 swap_header->info.nr_badpages - 1791 1 /* header page */; 1792 1793 if (nr_good_pages) { 1794 swap_map[0] = SWAP_MAP_BAD; 1795 p->max = maxpages; 1796 p->pages = nr_good_pages; 1797 nr_extents = setup_swap_extents(p, &span); 1798 if (nr_extents < 0) { 1799 error = nr_extents; 1800 goto bad_swap; 1801 } 1802 nr_good_pages = p->pages; 1803 } 1804 if (!nr_good_pages) { 1805 printk(KERN_WARNING "Empty swap-file\n"); 1806 error = -EINVAL; 1807 goto bad_swap; 1808 } 1809 1810 if (blk_queue_nonrot(bdev_get_queue(p->bdev))) { 1811 p->flags |= SWP_SOLIDSTATE; 1812 srandom32((u32)get_seconds()); 1813 p->cluster_next = 1 + (random32() % p->highest_bit); 1814 } 1815 if (discard_swap(p) == 0) 1816 p->flags |= SWP_DISCARDABLE; 1817 1818 mutex_lock(&swapon_mutex); 1819 spin_lock(&swap_lock); 1820 if (swap_flags & SWAP_FLAG_PREFER) 1821 p->prio = 1822 (swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT; 1823 else 1824 p->prio = --least_priority; 1825 p->swap_map = swap_map; 1826 p->flags |= SWP_WRITEOK; 1827 nr_swap_pages += nr_good_pages; 1828 total_swap_pages += nr_good_pages; 1829 1830 printk(KERN_INFO "Adding %uk swap on %s. " 1831 "Priority:%d extents:%d across:%lluk %s%s\n", 1832 nr_good_pages<<(PAGE_SHIFT-10), name, p->prio, 1833 nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10), 1834 (p->flags & SWP_SOLIDSTATE) ? "SS" : "", 1835 (p->flags & SWP_DISCARDABLE) ? "D" : ""); 1836 1837 /* insert swap space into swap_list: */ 1838 prev = -1; 1839 for (i = swap_list.head; i >= 0; i = swap_info[i].next) { 1840 if (p->prio >= swap_info[i].prio) { 1841 break; 1842 } 1843 prev = i; 1844 } 1845 p->next = i; 1846 if (prev < 0) { 1847 swap_list.head = swap_list.next = p - swap_info; 1848 } else { 1849 swap_info[prev].next = p - swap_info; 1850 } 1851 spin_unlock(&swap_lock); 1852 mutex_unlock(&swapon_mutex); 1853 error = 0; 1854 goto out; 1855bad_swap: 1856 if (bdev) { 1857 set_blocksize(bdev, p->old_block_size); 1858 bd_release(bdev); 1859 } 1860 destroy_swap_extents(p); 1861bad_swap_2: 1862 spin_lock(&swap_lock); 1863 p->swap_file = NULL; 1864 p->flags = 0; 1865 spin_unlock(&swap_lock); 1866 vfree(swap_map); 1867 if (swap_file) 1868 filp_close(swap_file, NULL); 1869out: 1870 if (page && !IS_ERR(page)) { 1871 kunmap(page); 1872 page_cache_release(page); 1873 } 1874 if (name) 1875 putname(name); 1876 if (did_down) { 1877 if (!error) 1878 inode->i_flags |= S_SWAPFILE; 1879 mutex_unlock(&inode->i_mutex); 1880 } 1881 return error; 1882} 1883 1884void si_swapinfo(struct sysinfo *val) 1885{ 1886 unsigned int i; 1887 unsigned long nr_to_be_unused = 0; 1888 1889 spin_lock(&swap_lock); 1890 for (i = 0; i < nr_swapfiles; i++) { 1891 if (!(swap_info[i].flags & SWP_USED) || 1892 (swap_info[i].flags & SWP_WRITEOK)) 1893 continue; 1894 nr_to_be_unused += swap_info[i].inuse_pages; 1895 } 1896 val->freeswap = nr_swap_pages + nr_to_be_unused; 1897 val->totalswap = total_swap_pages + nr_to_be_unused; 1898 spin_unlock(&swap_lock); 1899} 1900 1901/* 1902 * Verify that a swap entry is valid and increment its swap map count. 1903 * 1904 * Note: if swap_map[] reaches SWAP_MAP_MAX the entries are treated as 1905 * "permanent", but will be reclaimed by the next swapoff. 1906 */ 1907int swap_duplicate(swp_entry_t entry) 1908{ 1909 struct swap_info_struct * p; 1910 unsigned long offset, type; 1911 int result = 0; 1912 1913 if (is_migration_entry(entry)) 1914 return 1; 1915 1916 type = swp_type(entry); 1917 if (type >= nr_swapfiles) 1918 goto bad_file; 1919 p = type + swap_info; 1920 offset = swp_offset(entry); 1921 1922 spin_lock(&swap_lock); 1923 if (offset < p->max && p->swap_map[offset]) { 1924 if (p->swap_map[offset] < SWAP_MAP_MAX - 1) { 1925 p->swap_map[offset]++; 1926 result = 1; 1927 } else if (p->swap_map[offset] <= SWAP_MAP_MAX) { 1928 if (swap_overflow++ < 5) 1929 printk(KERN_WARNING "swap_dup: swap entry overflow\n"); 1930 p->swap_map[offset] = SWAP_MAP_MAX; 1931 result = 1; 1932 } 1933 } 1934 spin_unlock(&swap_lock); 1935out: 1936 return result; 1937 1938bad_file: 1939 printk(KERN_ERR "swap_dup: %s%08lx\n", Bad_file, entry.val); 1940 goto out; 1941} 1942 1943struct swap_info_struct * 1944get_swap_info_struct(unsigned type) 1945{ 1946 return &swap_info[type]; 1947} 1948 1949/* 1950 * swap_lock prevents swap_map being freed. Don't grab an extra 1951 * reference on the swaphandle, it doesn't matter if it becomes unused. 1952 */ 1953int valid_swaphandles(swp_entry_t entry, unsigned long *offset) 1954{ 1955 struct swap_info_struct *si; 1956 int our_page_cluster = page_cluster; 1957 pgoff_t target, toff; 1958 pgoff_t base, end; 1959 int nr_pages = 0; 1960 1961 if (!our_page_cluster) /* no readahead */ 1962 return 0; 1963 1964 si = &swap_info[swp_type(entry)]; 1965 target = swp_offset(entry); 1966 base = (target >> our_page_cluster) << our_page_cluster; 1967 end = base + (1 << our_page_cluster); 1968 if (!base) /* first page is swap header */ 1969 base++; 1970 1971 spin_lock(&swap_lock); 1972 if (end > si->max) /* don't go beyond end of map */ 1973 end = si->max; 1974 1975 /* Count contiguous allocated slots above our target */ 1976 for (toff = target; ++toff < end; nr_pages++) { 1977 /* Don't read in free or bad pages */ 1978 if (!si->swap_map[toff]) 1979 break; 1980 if (si->swap_map[toff] == SWAP_MAP_BAD) 1981 break; 1982 } 1983 /* Count contiguous allocated slots below our target */ 1984 for (toff = target; --toff >= base; nr_pages++) { 1985 /* Don't read in free or bad pages */ 1986 if (!si->swap_map[toff]) 1987 break; 1988 if (si->swap_map[toff] == SWAP_MAP_BAD) 1989 break; 1990 } 1991 spin_unlock(&swap_lock); 1992 1993 /* 1994 * Indicate starting offset, and return number of pages to get: 1995 * if only 1, say 0, since there's then no readahead to be done. 1996 */ 1997 *offset = ++toff; 1998 return nr_pages? ++nr_pages: 0; 1999} 2000