migrate.c revision 70384dc6dcc6aa76762200262820bdb8b724ecd5
1/* 2 * Memory Migration functionality - linux/mm/migration.c 3 * 4 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter 5 * 6 * Page migration was first developed in the context of the memory hotplug 7 * project. The main authors of the migration code are: 8 * 9 * IWAMOTO Toshihiro <iwamoto@valinux.co.jp> 10 * Hirokazu Takahashi <taka@valinux.co.jp> 11 * Dave Hansen <haveblue@us.ibm.com> 12 * Christoph Lameter 13 */ 14 15#include <linux/migrate.h> 16#include <linux/module.h> 17#include <linux/swap.h> 18#include <linux/swapops.h> 19#include <linux/pagemap.h> 20#include <linux/buffer_head.h> 21#include <linux/mm_inline.h> 22#include <linux/nsproxy.h> 23#include <linux/pagevec.h> 24#include <linux/ksm.h> 25#include <linux/rmap.h> 26#include <linux/topology.h> 27#include <linux/cpu.h> 28#include <linux/cpuset.h> 29#include <linux/writeback.h> 30#include <linux/mempolicy.h> 31#include <linux/vmalloc.h> 32#include <linux/security.h> 33#include <linux/memcontrol.h> 34#include <linux/syscalls.h> 35#include <linux/hugetlb.h> 36#include <linux/gfp.h> 37 38#include "internal.h" 39 40#define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru)) 41 42/* 43 * migrate_prep() needs to be called before we start compiling a list of pages 44 * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is 45 * undesirable, use migrate_prep_local() 46 */ 47int migrate_prep(void) 48{ 49 /* 50 * Clear the LRU lists so pages can be isolated. 51 * Note that pages may be moved off the LRU after we have 52 * drained them. Those pages will fail to migrate like other 53 * pages that may be busy. 54 */ 55 lru_add_drain_all(); 56 57 return 0; 58} 59 60/* Do the necessary work of migrate_prep but not if it involves other CPUs */ 61int migrate_prep_local(void) 62{ 63 lru_add_drain(); 64 65 return 0; 66} 67 68/* 69 * Add isolated pages on the list back to the LRU under page lock 70 * to avoid leaking evictable pages back onto unevictable list. 71 */ 72void putback_lru_pages(struct list_head *l) 73{ 74 struct page *page; 75 struct page *page2; 76 77 list_for_each_entry_safe(page, page2, l, lru) { 78 list_del(&page->lru); 79 dec_zone_page_state(page, NR_ISOLATED_ANON + 80 page_is_file_cache(page)); 81 putback_lru_page(page); 82 } 83} 84 85/* 86 * Restore a potential migration pte to a working pte entry 87 */ 88static int remove_migration_pte(struct page *new, struct vm_area_struct *vma, 89 unsigned long addr, void *old) 90{ 91 struct mm_struct *mm = vma->vm_mm; 92 swp_entry_t entry; 93 pgd_t *pgd; 94 pud_t *pud; 95 pmd_t *pmd; 96 pte_t *ptep, pte; 97 spinlock_t *ptl; 98 99 if (unlikely(PageHuge(new))) { 100 ptep = huge_pte_offset(mm, addr); 101 if (!ptep) 102 goto out; 103 ptl = &mm->page_table_lock; 104 } else { 105 pgd = pgd_offset(mm, addr); 106 if (!pgd_present(*pgd)) 107 goto out; 108 109 pud = pud_offset(pgd, addr); 110 if (!pud_present(*pud)) 111 goto out; 112 113 pmd = pmd_offset(pud, addr); 114 if (!pmd_present(*pmd)) 115 goto out; 116 117 ptep = pte_offset_map(pmd, addr); 118 119 if (!is_swap_pte(*ptep)) { 120 pte_unmap(ptep); 121 goto out; 122 } 123 124 ptl = pte_lockptr(mm, pmd); 125 } 126 127 spin_lock(ptl); 128 pte = *ptep; 129 if (!is_swap_pte(pte)) 130 goto unlock; 131 132 entry = pte_to_swp_entry(pte); 133 134 if (!is_migration_entry(entry) || 135 migration_entry_to_page(entry) != old) 136 goto unlock; 137 138 get_page(new); 139 pte = pte_mkold(mk_pte(new, vma->vm_page_prot)); 140 if (is_write_migration_entry(entry)) 141 pte = pte_mkwrite(pte); 142#ifdef CONFIG_HUGETLB_PAGE 143 if (PageHuge(new)) 144 pte = pte_mkhuge(pte); 145#endif 146 flush_cache_page(vma, addr, pte_pfn(pte)); 147 set_pte_at(mm, addr, ptep, pte); 148 149 if (PageHuge(new)) { 150 if (PageAnon(new)) 151 hugepage_add_anon_rmap(new, vma, addr); 152 else 153 page_dup_rmap(new); 154 } else if (PageAnon(new)) 155 page_add_anon_rmap(new, vma, addr); 156 else 157 page_add_file_rmap(new); 158 159 /* No need to invalidate - it was non-present before */ 160 update_mmu_cache(vma, addr, ptep); 161unlock: 162 pte_unmap_unlock(ptep, ptl); 163out: 164 return SWAP_AGAIN; 165} 166 167/* 168 * Get rid of all migration entries and replace them by 169 * references to the indicated page. 170 */ 171static void remove_migration_ptes(struct page *old, struct page *new) 172{ 173 rmap_walk(new, remove_migration_pte, old); 174} 175 176/* 177 * Something used the pte of a page under migration. We need to 178 * get to the page and wait until migration is finished. 179 * When we return from this function the fault will be retried. 180 * 181 * This function is called from do_swap_page(). 182 */ 183void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, 184 unsigned long address) 185{ 186 pte_t *ptep, pte; 187 spinlock_t *ptl; 188 swp_entry_t entry; 189 struct page *page; 190 191 ptep = pte_offset_map_lock(mm, pmd, address, &ptl); 192 pte = *ptep; 193 if (!is_swap_pte(pte)) 194 goto out; 195 196 entry = pte_to_swp_entry(pte); 197 if (!is_migration_entry(entry)) 198 goto out; 199 200 page = migration_entry_to_page(entry); 201 202 /* 203 * Once radix-tree replacement of page migration started, page_count 204 * *must* be zero. And, we don't want to call wait_on_page_locked() 205 * against a page without get_page(). 206 * So, we use get_page_unless_zero(), here. Even failed, page fault 207 * will occur again. 208 */ 209 if (!get_page_unless_zero(page)) 210 goto out; 211 pte_unmap_unlock(ptep, ptl); 212 wait_on_page_locked(page); 213 put_page(page); 214 return; 215out: 216 pte_unmap_unlock(ptep, ptl); 217} 218 219/* 220 * Replace the page in the mapping. 221 * 222 * The number of remaining references must be: 223 * 1 for anonymous pages without a mapping 224 * 2 for pages with a mapping 225 * 3 for pages with a mapping and PagePrivate/PagePrivate2 set. 226 */ 227static int migrate_page_move_mapping(struct address_space *mapping, 228 struct page *newpage, struct page *page) 229{ 230 int expected_count; 231 void **pslot; 232 233 if (!mapping) { 234 /* Anonymous page without mapping */ 235 if (page_count(page) != 1) 236 return -EAGAIN; 237 return 0; 238 } 239 240 spin_lock_irq(&mapping->tree_lock); 241 242 pslot = radix_tree_lookup_slot(&mapping->page_tree, 243 page_index(page)); 244 245 expected_count = 2 + page_has_private(page); 246 if (page_count(page) != expected_count || 247 (struct page *)radix_tree_deref_slot(pslot) != page) { 248 spin_unlock_irq(&mapping->tree_lock); 249 return -EAGAIN; 250 } 251 252 if (!page_freeze_refs(page, expected_count)) { 253 spin_unlock_irq(&mapping->tree_lock); 254 return -EAGAIN; 255 } 256 257 /* 258 * Now we know that no one else is looking at the page. 259 */ 260 get_page(newpage); /* add cache reference */ 261 if (PageSwapCache(page)) { 262 SetPageSwapCache(newpage); 263 set_page_private(newpage, page_private(page)); 264 } 265 266 radix_tree_replace_slot(pslot, newpage); 267 268 page_unfreeze_refs(page, expected_count); 269 /* 270 * Drop cache reference from old page. 271 * We know this isn't the last reference. 272 */ 273 __put_page(page); 274 275 /* 276 * If moved to a different zone then also account 277 * the page for that zone. Other VM counters will be 278 * taken care of when we establish references to the 279 * new page and drop references to the old page. 280 * 281 * Note that anonymous pages are accounted for 282 * via NR_FILE_PAGES and NR_ANON_PAGES if they 283 * are mapped to swap space. 284 */ 285 __dec_zone_page_state(page, NR_FILE_PAGES); 286 __inc_zone_page_state(newpage, NR_FILE_PAGES); 287 if (PageSwapBacked(page)) { 288 __dec_zone_page_state(page, NR_SHMEM); 289 __inc_zone_page_state(newpage, NR_SHMEM); 290 } 291 spin_unlock_irq(&mapping->tree_lock); 292 293 return 0; 294} 295 296/* 297 * The expected number of remaining references is the same as that 298 * of migrate_page_move_mapping(). 299 */ 300int migrate_huge_page_move_mapping(struct address_space *mapping, 301 struct page *newpage, struct page *page) 302{ 303 int expected_count; 304 void **pslot; 305 306 if (!mapping) { 307 if (page_count(page) != 1) 308 return -EAGAIN; 309 return 0; 310 } 311 312 spin_lock_irq(&mapping->tree_lock); 313 314 pslot = radix_tree_lookup_slot(&mapping->page_tree, 315 page_index(page)); 316 317 expected_count = 2 + page_has_private(page); 318 if (page_count(page) != expected_count || 319 (struct page *)radix_tree_deref_slot(pslot) != page) { 320 spin_unlock_irq(&mapping->tree_lock); 321 return -EAGAIN; 322 } 323 324 if (!page_freeze_refs(page, expected_count)) { 325 spin_unlock_irq(&mapping->tree_lock); 326 return -EAGAIN; 327 } 328 329 get_page(newpage); 330 331 radix_tree_replace_slot(pslot, newpage); 332 333 page_unfreeze_refs(page, expected_count); 334 335 __put_page(page); 336 337 spin_unlock_irq(&mapping->tree_lock); 338 return 0; 339} 340 341/* 342 * Copy the page to its new location 343 */ 344void migrate_page_copy(struct page *newpage, struct page *page) 345{ 346 if (PageHuge(page)) 347 copy_huge_page(newpage, page); 348 else 349 copy_highpage(newpage, page); 350 351 if (PageError(page)) 352 SetPageError(newpage); 353 if (PageReferenced(page)) 354 SetPageReferenced(newpage); 355 if (PageUptodate(page)) 356 SetPageUptodate(newpage); 357 if (TestClearPageActive(page)) { 358 VM_BUG_ON(PageUnevictable(page)); 359 SetPageActive(newpage); 360 } else if (TestClearPageUnevictable(page)) 361 SetPageUnevictable(newpage); 362 if (PageChecked(page)) 363 SetPageChecked(newpage); 364 if (PageMappedToDisk(page)) 365 SetPageMappedToDisk(newpage); 366 367 if (PageDirty(page)) { 368 clear_page_dirty_for_io(page); 369 /* 370 * Want to mark the page and the radix tree as dirty, and 371 * redo the accounting that clear_page_dirty_for_io undid, 372 * but we can't use set_page_dirty because that function 373 * is actually a signal that all of the page has become dirty. 374 * Wheras only part of our page may be dirty. 375 */ 376 __set_page_dirty_nobuffers(newpage); 377 } 378 379 mlock_migrate_page(newpage, page); 380 ksm_migrate_page(newpage, page); 381 382 ClearPageSwapCache(page); 383 ClearPagePrivate(page); 384 set_page_private(page, 0); 385 page->mapping = NULL; 386 387 /* 388 * If any waiters have accumulated on the new page then 389 * wake them up. 390 */ 391 if (PageWriteback(newpage)) 392 end_page_writeback(newpage); 393} 394 395/************************************************************ 396 * Migration functions 397 ***********************************************************/ 398 399/* Always fail migration. Used for mappings that are not movable */ 400int fail_migrate_page(struct address_space *mapping, 401 struct page *newpage, struct page *page) 402{ 403 return -EIO; 404} 405EXPORT_SYMBOL(fail_migrate_page); 406 407/* 408 * Common logic to directly migrate a single page suitable for 409 * pages that do not use PagePrivate/PagePrivate2. 410 * 411 * Pages are locked upon entry and exit. 412 */ 413int migrate_page(struct address_space *mapping, 414 struct page *newpage, struct page *page) 415{ 416 int rc; 417 418 BUG_ON(PageWriteback(page)); /* Writeback must be complete */ 419 420 rc = migrate_page_move_mapping(mapping, newpage, page); 421 422 if (rc) 423 return rc; 424 425 migrate_page_copy(newpage, page); 426 return 0; 427} 428EXPORT_SYMBOL(migrate_page); 429 430#ifdef CONFIG_BLOCK 431/* 432 * Migration function for pages with buffers. This function can only be used 433 * if the underlying filesystem guarantees that no other references to "page" 434 * exist. 435 */ 436int buffer_migrate_page(struct address_space *mapping, 437 struct page *newpage, struct page *page) 438{ 439 struct buffer_head *bh, *head; 440 int rc; 441 442 if (!page_has_buffers(page)) 443 return migrate_page(mapping, newpage, page); 444 445 head = page_buffers(page); 446 447 rc = migrate_page_move_mapping(mapping, newpage, page); 448 449 if (rc) 450 return rc; 451 452 bh = head; 453 do { 454 get_bh(bh); 455 lock_buffer(bh); 456 bh = bh->b_this_page; 457 458 } while (bh != head); 459 460 ClearPagePrivate(page); 461 set_page_private(newpage, page_private(page)); 462 set_page_private(page, 0); 463 put_page(page); 464 get_page(newpage); 465 466 bh = head; 467 do { 468 set_bh_page(bh, newpage, bh_offset(bh)); 469 bh = bh->b_this_page; 470 471 } while (bh != head); 472 473 SetPagePrivate(newpage); 474 475 migrate_page_copy(newpage, page); 476 477 bh = head; 478 do { 479 unlock_buffer(bh); 480 put_bh(bh); 481 bh = bh->b_this_page; 482 483 } while (bh != head); 484 485 return 0; 486} 487EXPORT_SYMBOL(buffer_migrate_page); 488#endif 489 490/* 491 * Writeback a page to clean the dirty state 492 */ 493static int writeout(struct address_space *mapping, struct page *page) 494{ 495 struct writeback_control wbc = { 496 .sync_mode = WB_SYNC_NONE, 497 .nr_to_write = 1, 498 .range_start = 0, 499 .range_end = LLONG_MAX, 500 .for_reclaim = 1 501 }; 502 int rc; 503 504 if (!mapping->a_ops->writepage) 505 /* No write method for the address space */ 506 return -EINVAL; 507 508 if (!clear_page_dirty_for_io(page)) 509 /* Someone else already triggered a write */ 510 return -EAGAIN; 511 512 /* 513 * A dirty page may imply that the underlying filesystem has 514 * the page on some queue. So the page must be clean for 515 * migration. Writeout may mean we loose the lock and the 516 * page state is no longer what we checked for earlier. 517 * At this point we know that the migration attempt cannot 518 * be successful. 519 */ 520 remove_migration_ptes(page, page); 521 522 rc = mapping->a_ops->writepage(page, &wbc); 523 524 if (rc != AOP_WRITEPAGE_ACTIVATE) 525 /* unlocked. Relock */ 526 lock_page(page); 527 528 return (rc < 0) ? -EIO : -EAGAIN; 529} 530 531/* 532 * Default handling if a filesystem does not provide a migration function. 533 */ 534static int fallback_migrate_page(struct address_space *mapping, 535 struct page *newpage, struct page *page) 536{ 537 if (PageDirty(page)) 538 return writeout(mapping, page); 539 540 /* 541 * Buffers may be managed in a filesystem specific way. 542 * We must have no buffers or drop them. 543 */ 544 if (page_has_private(page) && 545 !try_to_release_page(page, GFP_KERNEL)) 546 return -EAGAIN; 547 548 return migrate_page(mapping, newpage, page); 549} 550 551/* 552 * Move a page to a newly allocated page 553 * The page is locked and all ptes have been successfully removed. 554 * 555 * The new page will have replaced the old page if this function 556 * is successful. 557 * 558 * Return value: 559 * < 0 - error code 560 * == 0 - success 561 */ 562static int move_to_new_page(struct page *newpage, struct page *page, 563 int remap_swapcache) 564{ 565 struct address_space *mapping; 566 int rc; 567 568 /* 569 * Block others from accessing the page when we get around to 570 * establishing additional references. We are the only one 571 * holding a reference to the new page at this point. 572 */ 573 if (!trylock_page(newpage)) 574 BUG(); 575 576 /* Prepare mapping for the new page.*/ 577 newpage->index = page->index; 578 newpage->mapping = page->mapping; 579 if (PageSwapBacked(page)) 580 SetPageSwapBacked(newpage); 581 582 mapping = page_mapping(page); 583 if (!mapping) 584 rc = migrate_page(mapping, newpage, page); 585 else if (mapping->a_ops->migratepage) 586 /* 587 * Most pages have a mapping and most filesystems 588 * should provide a migration function. Anonymous 589 * pages are part of swap space which also has its 590 * own migration function. This is the most common 591 * path for page migration. 592 */ 593 rc = mapping->a_ops->migratepage(mapping, 594 newpage, page); 595 else 596 rc = fallback_migrate_page(mapping, newpage, page); 597 598 if (rc) { 599 newpage->mapping = NULL; 600 } else { 601 if (remap_swapcache) 602 remove_migration_ptes(page, newpage); 603 } 604 605 unlock_page(newpage); 606 607 return rc; 608} 609 610/* 611 * Obtain the lock on page, remove all ptes and migrate the page 612 * to the newly allocated page in newpage. 613 */ 614static int unmap_and_move(new_page_t get_new_page, unsigned long private, 615 struct page *page, int force, int offlining) 616{ 617 int rc = 0; 618 int *result = NULL; 619 struct page *newpage = get_new_page(page, private, &result); 620 int remap_swapcache = 1; 621 int rcu_locked = 0; 622 int charge = 0; 623 struct mem_cgroup *mem = NULL; 624 struct anon_vma *anon_vma = NULL; 625 626 if (!newpage) 627 return -ENOMEM; 628 629 if (page_count(page) == 1) { 630 /* page was freed from under us. So we are done. */ 631 goto move_newpage; 632 } 633 634 /* prepare cgroup just returns 0 or -ENOMEM */ 635 rc = -EAGAIN; 636 637 if (!trylock_page(page)) { 638 if (!force) 639 goto move_newpage; 640 lock_page(page); 641 } 642 643 /* 644 * Only memory hotplug's offline_pages() caller has locked out KSM, 645 * and can safely migrate a KSM page. The other cases have skipped 646 * PageKsm along with PageReserved - but it is only now when we have 647 * the page lock that we can be certain it will not go KSM beneath us 648 * (KSM will not upgrade a page from PageAnon to PageKsm when it sees 649 * its pagecount raised, but only here do we take the page lock which 650 * serializes that). 651 */ 652 if (PageKsm(page) && !offlining) { 653 rc = -EBUSY; 654 goto unlock; 655 } 656 657 /* charge against new page */ 658 charge = mem_cgroup_prepare_migration(page, newpage, &mem); 659 if (charge == -ENOMEM) { 660 rc = -ENOMEM; 661 goto unlock; 662 } 663 BUG_ON(charge); 664 665 if (PageWriteback(page)) { 666 if (!force) 667 goto uncharge; 668 wait_on_page_writeback(page); 669 } 670 /* 671 * By try_to_unmap(), page->mapcount goes down to 0 here. In this case, 672 * we cannot notice that anon_vma is freed while we migrates a page. 673 * This rcu_read_lock() delays freeing anon_vma pointer until the end 674 * of migration. File cache pages are no problem because of page_lock() 675 * File Caches may use write_page() or lock_page() in migration, then, 676 * just care Anon page here. 677 */ 678 if (PageAnon(page)) { 679 rcu_read_lock(); 680 rcu_locked = 1; 681 682 /* Determine how to safely use anon_vma */ 683 if (!page_mapped(page)) { 684 if (!PageSwapCache(page)) 685 goto rcu_unlock; 686 687 /* 688 * We cannot be sure that the anon_vma of an unmapped 689 * swapcache page is safe to use because we don't 690 * know in advance if the VMA that this page belonged 691 * to still exists. If the VMA and others sharing the 692 * data have been freed, then the anon_vma could 693 * already be invalid. 694 * 695 * To avoid this possibility, swapcache pages get 696 * migrated but are not remapped when migration 697 * completes 698 */ 699 remap_swapcache = 0; 700 } else { 701 /* 702 * Take a reference count on the anon_vma if the 703 * page is mapped so that it is guaranteed to 704 * exist when the page is remapped later 705 */ 706 anon_vma = page_anon_vma(page); 707 get_anon_vma(anon_vma); 708 } 709 } 710 711 /* 712 * Corner case handling: 713 * 1. When a new swap-cache page is read into, it is added to the LRU 714 * and treated as swapcache but it has no rmap yet. 715 * Calling try_to_unmap() against a page->mapping==NULL page will 716 * trigger a BUG. So handle it here. 717 * 2. An orphaned page (see truncate_complete_page) might have 718 * fs-private metadata. The page can be picked up due to memory 719 * offlining. Everywhere else except page reclaim, the page is 720 * invisible to the vm, so the page can not be migrated. So try to 721 * free the metadata, so the page can be freed. 722 */ 723 if (!page->mapping) { 724 if (!PageAnon(page) && page_has_private(page)) { 725 /* 726 * Go direct to try_to_free_buffers() here because 727 * a) that's what try_to_release_page() would do anyway 728 * b) we may be under rcu_read_lock() here, so we can't 729 * use GFP_KERNEL which is what try_to_release_page() 730 * needs to be effective. 731 */ 732 try_to_free_buffers(page); 733 goto rcu_unlock; 734 } 735 goto skip_unmap; 736 } 737 738 /* Establish migration ptes or remove ptes */ 739 try_to_unmap(page, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS); 740 741skip_unmap: 742 if (!page_mapped(page)) 743 rc = move_to_new_page(newpage, page, remap_swapcache); 744 745 if (rc && remap_swapcache) 746 remove_migration_ptes(page, page); 747rcu_unlock: 748 749 /* Drop an anon_vma reference if we took one */ 750 if (anon_vma) 751 drop_anon_vma(anon_vma); 752 753 if (rcu_locked) 754 rcu_read_unlock(); 755uncharge: 756 if (!charge) 757 mem_cgroup_end_migration(mem, page, newpage); 758unlock: 759 unlock_page(page); 760 761 if (rc != -EAGAIN) { 762 /* 763 * A page that has been migrated has all references 764 * removed and will be freed. A page that has not been 765 * migrated will have kepts its references and be 766 * restored. 767 */ 768 list_del(&page->lru); 769 dec_zone_page_state(page, NR_ISOLATED_ANON + 770 page_is_file_cache(page)); 771 putback_lru_page(page); 772 } 773 774move_newpage: 775 776 /* 777 * Move the new page to the LRU. If migration was not successful 778 * then this will free the page. 779 */ 780 putback_lru_page(newpage); 781 782 if (result) { 783 if (rc) 784 *result = rc; 785 else 786 *result = page_to_nid(newpage); 787 } 788 return rc; 789} 790 791/* 792 * Counterpart of unmap_and_move_page() for hugepage migration. 793 * 794 * This function doesn't wait the completion of hugepage I/O 795 * because there is no race between I/O and migration for hugepage. 796 * Note that currently hugepage I/O occurs only in direct I/O 797 * where no lock is held and PG_writeback is irrelevant, 798 * and writeback status of all subpages are counted in the reference 799 * count of the head page (i.e. if all subpages of a 2MB hugepage are 800 * under direct I/O, the reference of the head page is 512 and a bit more.) 801 * This means that when we try to migrate hugepage whose subpages are 802 * doing direct I/O, some references remain after try_to_unmap() and 803 * hugepage migration fails without data corruption. 804 * 805 * There is also no race when direct I/O is issued on the page under migration, 806 * because then pte is replaced with migration swap entry and direct I/O code 807 * will wait in the page fault for migration to complete. 808 */ 809static int unmap_and_move_huge_page(new_page_t get_new_page, 810 unsigned long private, struct page *hpage, 811 int force, int offlining) 812{ 813 int rc = 0; 814 int *result = NULL; 815 struct page *new_hpage = get_new_page(hpage, private, &result); 816 int rcu_locked = 0; 817 struct anon_vma *anon_vma = NULL; 818 819 if (!new_hpage) 820 return -ENOMEM; 821 822 rc = -EAGAIN; 823 824 if (!trylock_page(hpage)) { 825 if (!force) 826 goto out; 827 lock_page(hpage); 828 } 829 830 if (PageAnon(hpage)) { 831 rcu_read_lock(); 832 rcu_locked = 1; 833 834 if (page_mapped(hpage)) { 835 anon_vma = page_anon_vma(hpage); 836 atomic_inc(&anon_vma->external_refcount); 837 } 838 } 839 840 try_to_unmap(hpage, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS); 841 842 if (!page_mapped(hpage)) 843 rc = move_to_new_page(new_hpage, hpage, 1); 844 845 if (rc) 846 remove_migration_ptes(hpage, hpage); 847 848 if (anon_vma && atomic_dec_and_lock(&anon_vma->external_refcount, 849 &anon_vma->lock)) { 850 int empty = list_empty(&anon_vma->head); 851 spin_unlock(&anon_vma->lock); 852 if (empty) 853 anon_vma_free(anon_vma); 854 } 855 856 if (rcu_locked) 857 rcu_read_unlock(); 858out: 859 unlock_page(hpage); 860 861 if (rc != -EAGAIN) { 862 list_del(&hpage->lru); 863 put_page(hpage); 864 } 865 866 put_page(new_hpage); 867 868 if (result) { 869 if (rc) 870 *result = rc; 871 else 872 *result = page_to_nid(new_hpage); 873 } 874 return rc; 875} 876 877/* 878 * migrate_pages 879 * 880 * The function takes one list of pages to migrate and a function 881 * that determines from the page to be migrated and the private data 882 * the target of the move and allocates the page. 883 * 884 * The function returns after 10 attempts or if no pages 885 * are movable anymore because to has become empty 886 * or no retryable pages exist anymore. 887 * Caller should call putback_lru_pages to return pages to the LRU 888 * or free list. 889 * 890 * Return: Number of pages not migrated or error code. 891 */ 892int migrate_pages(struct list_head *from, 893 new_page_t get_new_page, unsigned long private, int offlining) 894{ 895 int retry = 1; 896 int nr_failed = 0; 897 int pass = 0; 898 struct page *page; 899 struct page *page2; 900 int swapwrite = current->flags & PF_SWAPWRITE; 901 int rc; 902 903 if (!swapwrite) 904 current->flags |= PF_SWAPWRITE; 905 906 for(pass = 0; pass < 10 && retry; pass++) { 907 retry = 0; 908 909 list_for_each_entry_safe(page, page2, from, lru) { 910 cond_resched(); 911 912 rc = unmap_and_move(get_new_page, private, 913 page, pass > 2, offlining); 914 915 switch(rc) { 916 case -ENOMEM: 917 goto out; 918 case -EAGAIN: 919 retry++; 920 break; 921 case 0: 922 break; 923 default: 924 /* Permanent failure */ 925 nr_failed++; 926 break; 927 } 928 } 929 } 930 rc = 0; 931out: 932 if (!swapwrite) 933 current->flags &= ~PF_SWAPWRITE; 934 935 if (rc) 936 return rc; 937 938 return nr_failed + retry; 939} 940 941int migrate_huge_pages(struct list_head *from, 942 new_page_t get_new_page, unsigned long private, int offlining) 943{ 944 int retry = 1; 945 int nr_failed = 0; 946 int pass = 0; 947 struct page *page; 948 struct page *page2; 949 int rc; 950 951 for (pass = 0; pass < 10 && retry; pass++) { 952 retry = 0; 953 954 list_for_each_entry_safe(page, page2, from, lru) { 955 cond_resched(); 956 957 rc = unmap_and_move_huge_page(get_new_page, 958 private, page, pass > 2, offlining); 959 960 switch(rc) { 961 case -ENOMEM: 962 goto out; 963 case -EAGAIN: 964 retry++; 965 break; 966 case 0: 967 break; 968 default: 969 /* Permanent failure */ 970 nr_failed++; 971 break; 972 } 973 } 974 } 975 rc = 0; 976out: 977 978 list_for_each_entry_safe(page, page2, from, lru) 979 put_page(page); 980 981 if (rc) 982 return rc; 983 984 return nr_failed + retry; 985} 986 987#ifdef CONFIG_NUMA 988/* 989 * Move a list of individual pages 990 */ 991struct page_to_node { 992 unsigned long addr; 993 struct page *page; 994 int node; 995 int status; 996}; 997 998static struct page *new_page_node(struct page *p, unsigned long private, 999 int **result) 1000{ 1001 struct page_to_node *pm = (struct page_to_node *)private; 1002 1003 while (pm->node != MAX_NUMNODES && pm->page != p) 1004 pm++; 1005 1006 if (pm->node == MAX_NUMNODES) 1007 return NULL; 1008 1009 *result = &pm->status; 1010 1011 return alloc_pages_exact_node(pm->node, 1012 GFP_HIGHUSER_MOVABLE | GFP_THISNODE, 0); 1013} 1014 1015/* 1016 * Move a set of pages as indicated in the pm array. The addr 1017 * field must be set to the virtual address of the page to be moved 1018 * and the node number must contain a valid target node. 1019 * The pm array ends with node = MAX_NUMNODES. 1020 */ 1021static int do_move_page_to_node_array(struct mm_struct *mm, 1022 struct page_to_node *pm, 1023 int migrate_all) 1024{ 1025 int err; 1026 struct page_to_node *pp; 1027 LIST_HEAD(pagelist); 1028 1029 down_read(&mm->mmap_sem); 1030 1031 /* 1032 * Build a list of pages to migrate 1033 */ 1034 for (pp = pm; pp->node != MAX_NUMNODES; pp++) { 1035 struct vm_area_struct *vma; 1036 struct page *page; 1037 1038 err = -EFAULT; 1039 vma = find_vma(mm, pp->addr); 1040 if (!vma || pp->addr < vma->vm_start || !vma_migratable(vma)) 1041 goto set_status; 1042 1043 page = follow_page(vma, pp->addr, FOLL_GET); 1044 1045 err = PTR_ERR(page); 1046 if (IS_ERR(page)) 1047 goto set_status; 1048 1049 err = -ENOENT; 1050 if (!page) 1051 goto set_status; 1052 1053 /* Use PageReserved to check for zero page */ 1054 if (PageReserved(page) || PageKsm(page)) 1055 goto put_and_set; 1056 1057 pp->page = page; 1058 err = page_to_nid(page); 1059 1060 if (err == pp->node) 1061 /* 1062 * Node already in the right place 1063 */ 1064 goto put_and_set; 1065 1066 err = -EACCES; 1067 if (page_mapcount(page) > 1 && 1068 !migrate_all) 1069 goto put_and_set; 1070 1071 err = isolate_lru_page(page); 1072 if (!err) { 1073 list_add_tail(&page->lru, &pagelist); 1074 inc_zone_page_state(page, NR_ISOLATED_ANON + 1075 page_is_file_cache(page)); 1076 } 1077put_and_set: 1078 /* 1079 * Either remove the duplicate refcount from 1080 * isolate_lru_page() or drop the page ref if it was 1081 * not isolated. 1082 */ 1083 put_page(page); 1084set_status: 1085 pp->status = err; 1086 } 1087 1088 err = 0; 1089 if (!list_empty(&pagelist)) { 1090 err = migrate_pages(&pagelist, new_page_node, 1091 (unsigned long)pm, 0); 1092 if (err) 1093 putback_lru_pages(&pagelist); 1094 } 1095 1096 up_read(&mm->mmap_sem); 1097 return err; 1098} 1099 1100/* 1101 * Migrate an array of page address onto an array of nodes and fill 1102 * the corresponding array of status. 1103 */ 1104static int do_pages_move(struct mm_struct *mm, struct task_struct *task, 1105 unsigned long nr_pages, 1106 const void __user * __user *pages, 1107 const int __user *nodes, 1108 int __user *status, int flags) 1109{ 1110 struct page_to_node *pm; 1111 nodemask_t task_nodes; 1112 unsigned long chunk_nr_pages; 1113 unsigned long chunk_start; 1114 int err; 1115 1116 task_nodes = cpuset_mems_allowed(task); 1117 1118 err = -ENOMEM; 1119 pm = (struct page_to_node *)__get_free_page(GFP_KERNEL); 1120 if (!pm) 1121 goto out; 1122 1123 migrate_prep(); 1124 1125 /* 1126 * Store a chunk of page_to_node array in a page, 1127 * but keep the last one as a marker 1128 */ 1129 chunk_nr_pages = (PAGE_SIZE / sizeof(struct page_to_node)) - 1; 1130 1131 for (chunk_start = 0; 1132 chunk_start < nr_pages; 1133 chunk_start += chunk_nr_pages) { 1134 int j; 1135 1136 if (chunk_start + chunk_nr_pages > nr_pages) 1137 chunk_nr_pages = nr_pages - chunk_start; 1138 1139 /* fill the chunk pm with addrs and nodes from user-space */ 1140 for (j = 0; j < chunk_nr_pages; j++) { 1141 const void __user *p; 1142 int node; 1143 1144 err = -EFAULT; 1145 if (get_user(p, pages + j + chunk_start)) 1146 goto out_pm; 1147 pm[j].addr = (unsigned long) p; 1148 1149 if (get_user(node, nodes + j + chunk_start)) 1150 goto out_pm; 1151 1152 err = -ENODEV; 1153 if (node < 0 || node >= MAX_NUMNODES) 1154 goto out_pm; 1155 1156 if (!node_state(node, N_HIGH_MEMORY)) 1157 goto out_pm; 1158 1159 err = -EACCES; 1160 if (!node_isset(node, task_nodes)) 1161 goto out_pm; 1162 1163 pm[j].node = node; 1164 } 1165 1166 /* End marker for this chunk */ 1167 pm[chunk_nr_pages].node = MAX_NUMNODES; 1168 1169 /* Migrate this chunk */ 1170 err = do_move_page_to_node_array(mm, pm, 1171 flags & MPOL_MF_MOVE_ALL); 1172 if (err < 0) 1173 goto out_pm; 1174 1175 /* Return status information */ 1176 for (j = 0; j < chunk_nr_pages; j++) 1177 if (put_user(pm[j].status, status + j + chunk_start)) { 1178 err = -EFAULT; 1179 goto out_pm; 1180 } 1181 } 1182 err = 0; 1183 1184out_pm: 1185 free_page((unsigned long)pm); 1186out: 1187 return err; 1188} 1189 1190/* 1191 * Determine the nodes of an array of pages and store it in an array of status. 1192 */ 1193static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages, 1194 const void __user **pages, int *status) 1195{ 1196 unsigned long i; 1197 1198 down_read(&mm->mmap_sem); 1199 1200 for (i = 0; i < nr_pages; i++) { 1201 unsigned long addr = (unsigned long)(*pages); 1202 struct vm_area_struct *vma; 1203 struct page *page; 1204 int err = -EFAULT; 1205 1206 vma = find_vma(mm, addr); 1207 if (!vma || addr < vma->vm_start) 1208 goto set_status; 1209 1210 page = follow_page(vma, addr, 0); 1211 1212 err = PTR_ERR(page); 1213 if (IS_ERR(page)) 1214 goto set_status; 1215 1216 err = -ENOENT; 1217 /* Use PageReserved to check for zero page */ 1218 if (!page || PageReserved(page) || PageKsm(page)) 1219 goto set_status; 1220 1221 err = page_to_nid(page); 1222set_status: 1223 *status = err; 1224 1225 pages++; 1226 status++; 1227 } 1228 1229 up_read(&mm->mmap_sem); 1230} 1231 1232/* 1233 * Determine the nodes of a user array of pages and store it in 1234 * a user array of status. 1235 */ 1236static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages, 1237 const void __user * __user *pages, 1238 int __user *status) 1239{ 1240#define DO_PAGES_STAT_CHUNK_NR 16 1241 const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR]; 1242 int chunk_status[DO_PAGES_STAT_CHUNK_NR]; 1243 1244 while (nr_pages) { 1245 unsigned long chunk_nr; 1246 1247 chunk_nr = nr_pages; 1248 if (chunk_nr > DO_PAGES_STAT_CHUNK_NR) 1249 chunk_nr = DO_PAGES_STAT_CHUNK_NR; 1250 1251 if (copy_from_user(chunk_pages, pages, chunk_nr * sizeof(*chunk_pages))) 1252 break; 1253 1254 do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status); 1255 1256 if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status))) 1257 break; 1258 1259 pages += chunk_nr; 1260 status += chunk_nr; 1261 nr_pages -= chunk_nr; 1262 } 1263 return nr_pages ? -EFAULT : 0; 1264} 1265 1266/* 1267 * Move a list of pages in the address space of the currently executing 1268 * process. 1269 */ 1270SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages, 1271 const void __user * __user *, pages, 1272 const int __user *, nodes, 1273 int __user *, status, int, flags) 1274{ 1275 const struct cred *cred = current_cred(), *tcred; 1276 struct task_struct *task; 1277 struct mm_struct *mm; 1278 int err; 1279 1280 /* Check flags */ 1281 if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL)) 1282 return -EINVAL; 1283 1284 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE)) 1285 return -EPERM; 1286 1287 /* Find the mm_struct */ 1288 read_lock(&tasklist_lock); 1289 task = pid ? find_task_by_vpid(pid) : current; 1290 if (!task) { 1291 read_unlock(&tasklist_lock); 1292 return -ESRCH; 1293 } 1294 mm = get_task_mm(task); 1295 read_unlock(&tasklist_lock); 1296 1297 if (!mm) 1298 return -EINVAL; 1299 1300 /* 1301 * Check if this process has the right to modify the specified 1302 * process. The right exists if the process has administrative 1303 * capabilities, superuser privileges or the same 1304 * userid as the target process. 1305 */ 1306 rcu_read_lock(); 1307 tcred = __task_cred(task); 1308 if (cred->euid != tcred->suid && cred->euid != tcred->uid && 1309 cred->uid != tcred->suid && cred->uid != tcred->uid && 1310 !capable(CAP_SYS_NICE)) { 1311 rcu_read_unlock(); 1312 err = -EPERM; 1313 goto out; 1314 } 1315 rcu_read_unlock(); 1316 1317 err = security_task_movememory(task); 1318 if (err) 1319 goto out; 1320 1321 if (nodes) { 1322 err = do_pages_move(mm, task, nr_pages, pages, nodes, status, 1323 flags); 1324 } else { 1325 err = do_pages_stat(mm, nr_pages, pages, status); 1326 } 1327 1328out: 1329 mmput(mm); 1330 return err; 1331} 1332 1333/* 1334 * Call migration functions in the vma_ops that may prepare 1335 * memory in a vm for migration. migration functions may perform 1336 * the migration for vmas that do not have an underlying page struct. 1337 */ 1338int migrate_vmas(struct mm_struct *mm, const nodemask_t *to, 1339 const nodemask_t *from, unsigned long flags) 1340{ 1341 struct vm_area_struct *vma; 1342 int err = 0; 1343 1344 for (vma = mm->mmap; vma && !err; vma = vma->vm_next) { 1345 if (vma->vm_ops && vma->vm_ops->migrate) { 1346 err = vma->vm_ops->migrate(vma, to, from, flags); 1347 if (err) 1348 break; 1349 } 1350 } 1351 return err; 1352} 1353#endif 1354