migrate.c revision dc386d4d1e98bb39fb967ee156cd456c802fc692
1/* 2 * Memory Migration functionality - linux/mm/migration.c 3 * 4 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter 5 * 6 * Page migration was first developed in the context of the memory hotplug 7 * project. The main authors of the migration code are: 8 * 9 * IWAMOTO Toshihiro <iwamoto@valinux.co.jp> 10 * Hirokazu Takahashi <taka@valinux.co.jp> 11 * Dave Hansen <haveblue@us.ibm.com> 12 * Christoph Lameter <clameter@sgi.com> 13 */ 14 15#include <linux/migrate.h> 16#include <linux/module.h> 17#include <linux/swap.h> 18#include <linux/swapops.h> 19#include <linux/pagemap.h> 20#include <linux/buffer_head.h> 21#include <linux/mm_inline.h> 22#include <linux/pagevec.h> 23#include <linux/rmap.h> 24#include <linux/topology.h> 25#include <linux/cpu.h> 26#include <linux/cpuset.h> 27#include <linux/writeback.h> 28#include <linux/mempolicy.h> 29#include <linux/vmalloc.h> 30#include <linux/security.h> 31 32#include "internal.h" 33 34#define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru)) 35 36/* 37 * Isolate one page from the LRU lists. If successful put it onto 38 * the indicated list with elevated page count. 39 * 40 * Result: 41 * -EBUSY: page not on LRU list 42 * 0: page removed from LRU list and added to the specified list. 43 */ 44int isolate_lru_page(struct page *page, struct list_head *pagelist) 45{ 46 int ret = -EBUSY; 47 48 if (PageLRU(page)) { 49 struct zone *zone = page_zone(page); 50 51 spin_lock_irq(&zone->lru_lock); 52 if (PageLRU(page)) { 53 ret = 0; 54 get_page(page); 55 ClearPageLRU(page); 56 if (PageActive(page)) 57 del_page_from_active_list(zone, page); 58 else 59 del_page_from_inactive_list(zone, page); 60 list_add_tail(&page->lru, pagelist); 61 } 62 spin_unlock_irq(&zone->lru_lock); 63 } 64 return ret; 65} 66 67/* 68 * migrate_prep() needs to be called before we start compiling a list of pages 69 * to be migrated using isolate_lru_page(). 70 */ 71int migrate_prep(void) 72{ 73 /* 74 * Clear the LRU lists so pages can be isolated. 75 * Note that pages may be moved off the LRU after we have 76 * drained them. Those pages will fail to migrate like other 77 * pages that may be busy. 78 */ 79 lru_add_drain_all(); 80 81 return 0; 82} 83 84static inline void move_to_lru(struct page *page) 85{ 86 if (PageActive(page)) { 87 /* 88 * lru_cache_add_active checks that 89 * the PG_active bit is off. 90 */ 91 ClearPageActive(page); 92 lru_cache_add_active(page); 93 } else { 94 lru_cache_add(page); 95 } 96 put_page(page); 97} 98 99/* 100 * Add isolated pages on the list back to the LRU. 101 * 102 * returns the number of pages put back. 103 */ 104int putback_lru_pages(struct list_head *l) 105{ 106 struct page *page; 107 struct page *page2; 108 int count = 0; 109 110 list_for_each_entry_safe(page, page2, l, lru) { 111 list_del(&page->lru); 112 move_to_lru(page); 113 count++; 114 } 115 return count; 116} 117 118static inline int is_swap_pte(pte_t pte) 119{ 120 return !pte_none(pte) && !pte_present(pte) && !pte_file(pte); 121} 122 123/* 124 * Restore a potential migration pte to a working pte entry 125 */ 126static void remove_migration_pte(struct vm_area_struct *vma, 127 struct page *old, struct page *new) 128{ 129 struct mm_struct *mm = vma->vm_mm; 130 swp_entry_t entry; 131 pgd_t *pgd; 132 pud_t *pud; 133 pmd_t *pmd; 134 pte_t *ptep, pte; 135 spinlock_t *ptl; 136 unsigned long addr = page_address_in_vma(new, vma); 137 138 if (addr == -EFAULT) 139 return; 140 141 pgd = pgd_offset(mm, addr); 142 if (!pgd_present(*pgd)) 143 return; 144 145 pud = pud_offset(pgd, addr); 146 if (!pud_present(*pud)) 147 return; 148 149 pmd = pmd_offset(pud, addr); 150 if (!pmd_present(*pmd)) 151 return; 152 153 ptep = pte_offset_map(pmd, addr); 154 155 if (!is_swap_pte(*ptep)) { 156 pte_unmap(ptep); 157 return; 158 } 159 160 ptl = pte_lockptr(mm, pmd); 161 spin_lock(ptl); 162 pte = *ptep; 163 if (!is_swap_pte(pte)) 164 goto out; 165 166 entry = pte_to_swp_entry(pte); 167 168 if (!is_migration_entry(entry) || migration_entry_to_page(entry) != old) 169 goto out; 170 171 get_page(new); 172 pte = pte_mkold(mk_pte(new, vma->vm_page_prot)); 173 if (is_write_migration_entry(entry)) 174 pte = pte_mkwrite(pte); 175 set_pte_at(mm, addr, ptep, pte); 176 177 if (PageAnon(new)) 178 page_add_anon_rmap(new, vma, addr); 179 else 180 page_add_file_rmap(new); 181 182 /* No need to invalidate - it was non-present before */ 183 update_mmu_cache(vma, addr, pte); 184 lazy_mmu_prot_update(pte); 185 186out: 187 pte_unmap_unlock(ptep, ptl); 188} 189 190/* 191 * Note that remove_file_migration_ptes will only work on regular mappings, 192 * Nonlinear mappings do not use migration entries. 193 */ 194static void remove_file_migration_ptes(struct page *old, struct page *new) 195{ 196 struct vm_area_struct *vma; 197 struct address_space *mapping = page_mapping(new); 198 struct prio_tree_iter iter; 199 pgoff_t pgoff = new->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); 200 201 if (!mapping) 202 return; 203 204 spin_lock(&mapping->i_mmap_lock); 205 206 vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) 207 remove_migration_pte(vma, old, new); 208 209 spin_unlock(&mapping->i_mmap_lock); 210} 211 212/* 213 * Must hold mmap_sem lock on at least one of the vmas containing 214 * the page so that the anon_vma cannot vanish. 215 */ 216static void remove_anon_migration_ptes(struct page *old, struct page *new) 217{ 218 struct anon_vma *anon_vma; 219 struct vm_area_struct *vma; 220 unsigned long mapping; 221 222 mapping = (unsigned long)new->mapping; 223 224 if (!mapping || (mapping & PAGE_MAPPING_ANON) == 0) 225 return; 226 227 /* 228 * We hold the mmap_sem lock. So no need to call page_lock_anon_vma. 229 */ 230 anon_vma = (struct anon_vma *) (mapping - PAGE_MAPPING_ANON); 231 spin_lock(&anon_vma->lock); 232 233 list_for_each_entry(vma, &anon_vma->head, anon_vma_node) 234 remove_migration_pte(vma, old, new); 235 236 spin_unlock(&anon_vma->lock); 237} 238 239/* 240 * Get rid of all migration entries and replace them by 241 * references to the indicated page. 242 */ 243static void remove_migration_ptes(struct page *old, struct page *new) 244{ 245 if (PageAnon(new)) 246 remove_anon_migration_ptes(old, new); 247 else 248 remove_file_migration_ptes(old, new); 249} 250 251/* 252 * Something used the pte of a page under migration. We need to 253 * get to the page and wait until migration is finished. 254 * When we return from this function the fault will be retried. 255 * 256 * This function is called from do_swap_page(). 257 */ 258void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, 259 unsigned long address) 260{ 261 pte_t *ptep, pte; 262 spinlock_t *ptl; 263 swp_entry_t entry; 264 struct page *page; 265 266 ptep = pte_offset_map_lock(mm, pmd, address, &ptl); 267 pte = *ptep; 268 if (!is_swap_pte(pte)) 269 goto out; 270 271 entry = pte_to_swp_entry(pte); 272 if (!is_migration_entry(entry)) 273 goto out; 274 275 page = migration_entry_to_page(entry); 276 277 get_page(page); 278 pte_unmap_unlock(ptep, ptl); 279 wait_on_page_locked(page); 280 put_page(page); 281 return; 282out: 283 pte_unmap_unlock(ptep, ptl); 284} 285 286/* 287 * Replace the page in the mapping. 288 * 289 * The number of remaining references must be: 290 * 1 for anonymous pages without a mapping 291 * 2 for pages with a mapping 292 * 3 for pages with a mapping and PagePrivate set. 293 */ 294static int migrate_page_move_mapping(struct address_space *mapping, 295 struct page *newpage, struct page *page) 296{ 297 void **pslot; 298 299 if (!mapping) { 300 /* Anonymous page without mapping */ 301 if (page_count(page) != 1) 302 return -EAGAIN; 303 return 0; 304 } 305 306 write_lock_irq(&mapping->tree_lock); 307 308 pslot = radix_tree_lookup_slot(&mapping->page_tree, 309 page_index(page)); 310 311 if (page_count(page) != 2 + !!PagePrivate(page) || 312 (struct page *)radix_tree_deref_slot(pslot) != page) { 313 write_unlock_irq(&mapping->tree_lock); 314 return -EAGAIN; 315 } 316 317 /* 318 * Now we know that no one else is looking at the page. 319 */ 320 get_page(newpage); /* add cache reference */ 321#ifdef CONFIG_SWAP 322 if (PageSwapCache(page)) { 323 SetPageSwapCache(newpage); 324 set_page_private(newpage, page_private(page)); 325 } 326#endif 327 328 radix_tree_replace_slot(pslot, newpage); 329 330 /* 331 * Drop cache reference from old page. 332 * We know this isn't the last reference. 333 */ 334 __put_page(page); 335 336 /* 337 * If moved to a different zone then also account 338 * the page for that zone. Other VM counters will be 339 * taken care of when we establish references to the 340 * new page and drop references to the old page. 341 * 342 * Note that anonymous pages are accounted for 343 * via NR_FILE_PAGES and NR_ANON_PAGES if they 344 * are mapped to swap space. 345 */ 346 __dec_zone_page_state(page, NR_FILE_PAGES); 347 __inc_zone_page_state(newpage, NR_FILE_PAGES); 348 349 write_unlock_irq(&mapping->tree_lock); 350 351 return 0; 352} 353 354/* 355 * Copy the page to its new location 356 */ 357static void migrate_page_copy(struct page *newpage, struct page *page) 358{ 359 copy_highpage(newpage, page); 360 361 if (PageError(page)) 362 SetPageError(newpage); 363 if (PageReferenced(page)) 364 SetPageReferenced(newpage); 365 if (PageUptodate(page)) 366 SetPageUptodate(newpage); 367 if (PageActive(page)) 368 SetPageActive(newpage); 369 if (PageChecked(page)) 370 SetPageChecked(newpage); 371 if (PageMappedToDisk(page)) 372 SetPageMappedToDisk(newpage); 373 374 if (PageDirty(page)) { 375 clear_page_dirty_for_io(page); 376 set_page_dirty(newpage); 377 } 378 379#ifdef CONFIG_SWAP 380 ClearPageSwapCache(page); 381#endif 382 ClearPageActive(page); 383 ClearPagePrivate(page); 384 set_page_private(page, 0); 385 page->mapping = NULL; 386 387 /* 388 * If any waiters have accumulated on the new page then 389 * wake them up. 390 */ 391 if (PageWriteback(newpage)) 392 end_page_writeback(newpage); 393} 394 395/************************************************************ 396 * Migration functions 397 ***********************************************************/ 398 399/* Always fail migration. Used for mappings that are not movable */ 400int fail_migrate_page(struct address_space *mapping, 401 struct page *newpage, struct page *page) 402{ 403 return -EIO; 404} 405EXPORT_SYMBOL(fail_migrate_page); 406 407/* 408 * Common logic to directly migrate a single page suitable for 409 * pages that do not use PagePrivate. 410 * 411 * Pages are locked upon entry and exit. 412 */ 413int migrate_page(struct address_space *mapping, 414 struct page *newpage, struct page *page) 415{ 416 int rc; 417 418 BUG_ON(PageWriteback(page)); /* Writeback must be complete */ 419 420 rc = migrate_page_move_mapping(mapping, newpage, page); 421 422 if (rc) 423 return rc; 424 425 migrate_page_copy(newpage, page); 426 return 0; 427} 428EXPORT_SYMBOL(migrate_page); 429 430#ifdef CONFIG_BLOCK 431/* 432 * Migration function for pages with buffers. This function can only be used 433 * if the underlying filesystem guarantees that no other references to "page" 434 * exist. 435 */ 436int buffer_migrate_page(struct address_space *mapping, 437 struct page *newpage, struct page *page) 438{ 439 struct buffer_head *bh, *head; 440 int rc; 441 442 if (!page_has_buffers(page)) 443 return migrate_page(mapping, newpage, page); 444 445 head = page_buffers(page); 446 447 rc = migrate_page_move_mapping(mapping, newpage, page); 448 449 if (rc) 450 return rc; 451 452 bh = head; 453 do { 454 get_bh(bh); 455 lock_buffer(bh); 456 bh = bh->b_this_page; 457 458 } while (bh != head); 459 460 ClearPagePrivate(page); 461 set_page_private(newpage, page_private(page)); 462 set_page_private(page, 0); 463 put_page(page); 464 get_page(newpage); 465 466 bh = head; 467 do { 468 set_bh_page(bh, newpage, bh_offset(bh)); 469 bh = bh->b_this_page; 470 471 } while (bh != head); 472 473 SetPagePrivate(newpage); 474 475 migrate_page_copy(newpage, page); 476 477 bh = head; 478 do { 479 unlock_buffer(bh); 480 put_bh(bh); 481 bh = bh->b_this_page; 482 483 } while (bh != head); 484 485 return 0; 486} 487EXPORT_SYMBOL(buffer_migrate_page); 488#endif 489 490/* 491 * Writeback a page to clean the dirty state 492 */ 493static int writeout(struct address_space *mapping, struct page *page) 494{ 495 struct writeback_control wbc = { 496 .sync_mode = WB_SYNC_NONE, 497 .nr_to_write = 1, 498 .range_start = 0, 499 .range_end = LLONG_MAX, 500 .nonblocking = 1, 501 .for_reclaim = 1 502 }; 503 int rc; 504 505 if (!mapping->a_ops->writepage) 506 /* No write method for the address space */ 507 return -EINVAL; 508 509 if (!clear_page_dirty_for_io(page)) 510 /* Someone else already triggered a write */ 511 return -EAGAIN; 512 513 /* 514 * A dirty page may imply that the underlying filesystem has 515 * the page on some queue. So the page must be clean for 516 * migration. Writeout may mean we loose the lock and the 517 * page state is no longer what we checked for earlier. 518 * At this point we know that the migration attempt cannot 519 * be successful. 520 */ 521 remove_migration_ptes(page, page); 522 523 rc = mapping->a_ops->writepage(page, &wbc); 524 if (rc < 0) 525 /* I/O Error writing */ 526 return -EIO; 527 528 if (rc != AOP_WRITEPAGE_ACTIVATE) 529 /* unlocked. Relock */ 530 lock_page(page); 531 532 return -EAGAIN; 533} 534 535/* 536 * Default handling if a filesystem does not provide a migration function. 537 */ 538static int fallback_migrate_page(struct address_space *mapping, 539 struct page *newpage, struct page *page) 540{ 541 if (PageDirty(page)) 542 return writeout(mapping, page); 543 544 /* 545 * Buffers may be managed in a filesystem specific way. 546 * We must have no buffers or drop them. 547 */ 548 if (PagePrivate(page) && 549 !try_to_release_page(page, GFP_KERNEL)) 550 return -EAGAIN; 551 552 return migrate_page(mapping, newpage, page); 553} 554 555/* 556 * Move a page to a newly allocated page 557 * The page is locked and all ptes have been successfully removed. 558 * 559 * The new page will have replaced the old page if this function 560 * is successful. 561 */ 562static int move_to_new_page(struct page *newpage, struct page *page) 563{ 564 struct address_space *mapping; 565 int rc; 566 567 /* 568 * Block others from accessing the page when we get around to 569 * establishing additional references. We are the only one 570 * holding a reference to the new page at this point. 571 */ 572 if (TestSetPageLocked(newpage)) 573 BUG(); 574 575 /* Prepare mapping for the new page.*/ 576 newpage->index = page->index; 577 newpage->mapping = page->mapping; 578 579 mapping = page_mapping(page); 580 if (!mapping) 581 rc = migrate_page(mapping, newpage, page); 582 else if (mapping->a_ops->migratepage) 583 /* 584 * Most pages have a mapping and most filesystems 585 * should provide a migration function. Anonymous 586 * pages are part of swap space which also has its 587 * own migration function. This is the most common 588 * path for page migration. 589 */ 590 rc = mapping->a_ops->migratepage(mapping, 591 newpage, page); 592 else 593 rc = fallback_migrate_page(mapping, newpage, page); 594 595 if (!rc) 596 remove_migration_ptes(page, newpage); 597 else 598 newpage->mapping = NULL; 599 600 unlock_page(newpage); 601 602 return rc; 603} 604 605/* 606 * Obtain the lock on page, remove all ptes and migrate the page 607 * to the newly allocated page in newpage. 608 */ 609static int unmap_and_move(new_page_t get_new_page, unsigned long private, 610 struct page *page, int force) 611{ 612 int rc = 0; 613 int *result = NULL; 614 struct page *newpage = get_new_page(page, private, &result); 615 616 if (!newpage) 617 return -ENOMEM; 618 619 if (page_count(page) == 1) 620 /* page was freed from under us. So we are done. */ 621 goto move_newpage; 622 623 rc = -EAGAIN; 624 if (TestSetPageLocked(page)) { 625 if (!force) 626 goto move_newpage; 627 lock_page(page); 628 } 629 630 if (PageWriteback(page)) { 631 if (!force) 632 goto unlock; 633 wait_on_page_writeback(page); 634 } 635 /* 636 * By try_to_unmap(), page->mapcount goes down to 0 here. In this case, 637 * we cannot notice that anon_vma is freed while we migrates a page. 638 * This rcu_read_lock() delays freeing anon_vma pointer until the end 639 * of migration. File cache pages are no problem because of page_lock() 640 */ 641 rcu_read_lock(); 642 /* 643 * This is a corner case handling. 644 * When a new swap-cache is read into, it is linked to LRU 645 * and treated as swapcache but has no rmap yet. 646 * Calling try_to_unmap() against a page->mapping==NULL page is 647 * BUG. So handle it here. 648 */ 649 if (!page->mapping) 650 goto rcu_unlock; 651 /* Establish migration ptes or remove ptes */ 652 try_to_unmap(page, 1); 653 654 if (!page_mapped(page)) 655 rc = move_to_new_page(newpage, page); 656 657 if (rc) 658 remove_migration_ptes(page, page); 659rcu_unlock: 660 rcu_read_unlock(); 661 662unlock: 663 664 unlock_page(page); 665 666 if (rc != -EAGAIN) { 667 /* 668 * A page that has been migrated has all references 669 * removed and will be freed. A page that has not been 670 * migrated will have kepts its references and be 671 * restored. 672 */ 673 list_del(&page->lru); 674 move_to_lru(page); 675 } 676 677move_newpage: 678 /* 679 * Move the new page to the LRU. If migration was not successful 680 * then this will free the page. 681 */ 682 move_to_lru(newpage); 683 if (result) { 684 if (rc) 685 *result = rc; 686 else 687 *result = page_to_nid(newpage); 688 } 689 return rc; 690} 691 692/* 693 * migrate_pages 694 * 695 * The function takes one list of pages to migrate and a function 696 * that determines from the page to be migrated and the private data 697 * the target of the move and allocates the page. 698 * 699 * The function returns after 10 attempts or if no pages 700 * are movable anymore because to has become empty 701 * or no retryable pages exist anymore. All pages will be 702 * retruned to the LRU or freed. 703 * 704 * Return: Number of pages not migrated or error code. 705 */ 706int migrate_pages(struct list_head *from, 707 new_page_t get_new_page, unsigned long private) 708{ 709 int retry = 1; 710 int nr_failed = 0; 711 int pass = 0; 712 struct page *page; 713 struct page *page2; 714 int swapwrite = current->flags & PF_SWAPWRITE; 715 int rc; 716 717 if (!swapwrite) 718 current->flags |= PF_SWAPWRITE; 719 720 for(pass = 0; pass < 10 && retry; pass++) { 721 retry = 0; 722 723 list_for_each_entry_safe(page, page2, from, lru) { 724 cond_resched(); 725 726 rc = unmap_and_move(get_new_page, private, 727 page, pass > 2); 728 729 switch(rc) { 730 case -ENOMEM: 731 goto out; 732 case -EAGAIN: 733 retry++; 734 break; 735 case 0: 736 break; 737 default: 738 /* Permanent failure */ 739 nr_failed++; 740 break; 741 } 742 } 743 } 744 rc = 0; 745out: 746 if (!swapwrite) 747 current->flags &= ~PF_SWAPWRITE; 748 749 putback_lru_pages(from); 750 751 if (rc) 752 return rc; 753 754 return nr_failed + retry; 755} 756 757#ifdef CONFIG_NUMA 758/* 759 * Move a list of individual pages 760 */ 761struct page_to_node { 762 unsigned long addr; 763 struct page *page; 764 int node; 765 int status; 766}; 767 768static struct page *new_page_node(struct page *p, unsigned long private, 769 int **result) 770{ 771 struct page_to_node *pm = (struct page_to_node *)private; 772 773 while (pm->node != MAX_NUMNODES && pm->page != p) 774 pm++; 775 776 if (pm->node == MAX_NUMNODES) 777 return NULL; 778 779 *result = &pm->status; 780 781 return alloc_pages_node(pm->node, 782 GFP_HIGHUSER_MOVABLE | GFP_THISNODE, 0); 783} 784 785/* 786 * Move a set of pages as indicated in the pm array. The addr 787 * field must be set to the virtual address of the page to be moved 788 * and the node number must contain a valid target node. 789 */ 790static int do_move_pages(struct mm_struct *mm, struct page_to_node *pm, 791 int migrate_all) 792{ 793 int err; 794 struct page_to_node *pp; 795 LIST_HEAD(pagelist); 796 797 down_read(&mm->mmap_sem); 798 799 /* 800 * Build a list of pages to migrate 801 */ 802 migrate_prep(); 803 for (pp = pm; pp->node != MAX_NUMNODES; pp++) { 804 struct vm_area_struct *vma; 805 struct page *page; 806 807 /* 808 * A valid page pointer that will not match any of the 809 * pages that will be moved. 810 */ 811 pp->page = ZERO_PAGE(0); 812 813 err = -EFAULT; 814 vma = find_vma(mm, pp->addr); 815 if (!vma || !vma_migratable(vma)) 816 goto set_status; 817 818 page = follow_page(vma, pp->addr, FOLL_GET); 819 err = -ENOENT; 820 if (!page) 821 goto set_status; 822 823 if (PageReserved(page)) /* Check for zero page */ 824 goto put_and_set; 825 826 pp->page = page; 827 err = page_to_nid(page); 828 829 if (err == pp->node) 830 /* 831 * Node already in the right place 832 */ 833 goto put_and_set; 834 835 err = -EACCES; 836 if (page_mapcount(page) > 1 && 837 !migrate_all) 838 goto put_and_set; 839 840 err = isolate_lru_page(page, &pagelist); 841put_and_set: 842 /* 843 * Either remove the duplicate refcount from 844 * isolate_lru_page() or drop the page ref if it was 845 * not isolated. 846 */ 847 put_page(page); 848set_status: 849 pp->status = err; 850 } 851 852 if (!list_empty(&pagelist)) 853 err = migrate_pages(&pagelist, new_page_node, 854 (unsigned long)pm); 855 else 856 err = -ENOENT; 857 858 up_read(&mm->mmap_sem); 859 return err; 860} 861 862/* 863 * Determine the nodes of a list of pages. The addr in the pm array 864 * must have been set to the virtual address of which we want to determine 865 * the node number. 866 */ 867static int do_pages_stat(struct mm_struct *mm, struct page_to_node *pm) 868{ 869 down_read(&mm->mmap_sem); 870 871 for ( ; pm->node != MAX_NUMNODES; pm++) { 872 struct vm_area_struct *vma; 873 struct page *page; 874 int err; 875 876 err = -EFAULT; 877 vma = find_vma(mm, pm->addr); 878 if (!vma) 879 goto set_status; 880 881 page = follow_page(vma, pm->addr, 0); 882 err = -ENOENT; 883 /* Use PageReserved to check for zero page */ 884 if (!page || PageReserved(page)) 885 goto set_status; 886 887 err = page_to_nid(page); 888set_status: 889 pm->status = err; 890 } 891 892 up_read(&mm->mmap_sem); 893 return 0; 894} 895 896/* 897 * Move a list of pages in the address space of the currently executing 898 * process. 899 */ 900asmlinkage long sys_move_pages(pid_t pid, unsigned long nr_pages, 901 const void __user * __user *pages, 902 const int __user *nodes, 903 int __user *status, int flags) 904{ 905 int err = 0; 906 int i; 907 struct task_struct *task; 908 nodemask_t task_nodes; 909 struct mm_struct *mm; 910 struct page_to_node *pm = NULL; 911 912 /* Check flags */ 913 if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL)) 914 return -EINVAL; 915 916 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE)) 917 return -EPERM; 918 919 /* Find the mm_struct */ 920 read_lock(&tasklist_lock); 921 task = pid ? find_task_by_pid(pid) : current; 922 if (!task) { 923 read_unlock(&tasklist_lock); 924 return -ESRCH; 925 } 926 mm = get_task_mm(task); 927 read_unlock(&tasklist_lock); 928 929 if (!mm) 930 return -EINVAL; 931 932 /* 933 * Check if this process has the right to modify the specified 934 * process. The right exists if the process has administrative 935 * capabilities, superuser privileges or the same 936 * userid as the target process. 937 */ 938 if ((current->euid != task->suid) && (current->euid != task->uid) && 939 (current->uid != task->suid) && (current->uid != task->uid) && 940 !capable(CAP_SYS_NICE)) { 941 err = -EPERM; 942 goto out2; 943 } 944 945 err = security_task_movememory(task); 946 if (err) 947 goto out2; 948 949 950 task_nodes = cpuset_mems_allowed(task); 951 952 /* Limit nr_pages so that the multiplication may not overflow */ 953 if (nr_pages >= ULONG_MAX / sizeof(struct page_to_node) - 1) { 954 err = -E2BIG; 955 goto out2; 956 } 957 958 pm = vmalloc((nr_pages + 1) * sizeof(struct page_to_node)); 959 if (!pm) { 960 err = -ENOMEM; 961 goto out2; 962 } 963 964 /* 965 * Get parameters from user space and initialize the pm 966 * array. Return various errors if the user did something wrong. 967 */ 968 for (i = 0; i < nr_pages; i++) { 969 const void *p; 970 971 err = -EFAULT; 972 if (get_user(p, pages + i)) 973 goto out; 974 975 pm[i].addr = (unsigned long)p; 976 if (nodes) { 977 int node; 978 979 if (get_user(node, nodes + i)) 980 goto out; 981 982 err = -ENODEV; 983 if (!node_online(node)) 984 goto out; 985 986 err = -EACCES; 987 if (!node_isset(node, task_nodes)) 988 goto out; 989 990 pm[i].node = node; 991 } else 992 pm[i].node = 0; /* anything to not match MAX_NUMNODES */ 993 } 994 /* End marker */ 995 pm[nr_pages].node = MAX_NUMNODES; 996 997 if (nodes) 998 err = do_move_pages(mm, pm, flags & MPOL_MF_MOVE_ALL); 999 else 1000 err = do_pages_stat(mm, pm); 1001 1002 if (err >= 0) 1003 /* Return status information */ 1004 for (i = 0; i < nr_pages; i++) 1005 if (put_user(pm[i].status, status + i)) 1006 err = -EFAULT; 1007 1008out: 1009 vfree(pm); 1010out2: 1011 mmput(mm); 1012 return err; 1013} 1014#endif 1015 1016/* 1017 * Call migration functions in the vma_ops that may prepare 1018 * memory in a vm for migration. migration functions may perform 1019 * the migration for vmas that do not have an underlying page struct. 1020 */ 1021int migrate_vmas(struct mm_struct *mm, const nodemask_t *to, 1022 const nodemask_t *from, unsigned long flags) 1023{ 1024 struct vm_area_struct *vma; 1025 int err = 0; 1026 1027 for(vma = mm->mmap; vma->vm_next && !err; vma = vma->vm_next) { 1028 if (vma->vm_ops && vma->vm_ops->migrate) { 1029 err = vma->vm_ops->migrate(vma, to, from, flags); 1030 if (err) 1031 break; 1032 } 1033 } 1034 return err; 1035} 1036