mempolicy.c revision 0def08e3acc2c9c934e4671487029aed52202d42
1/* 2 * Simple NUMA memory policy for the Linux kernel. 3 * 4 * Copyright 2003,2004 Andi Kleen, SuSE Labs. 5 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc. 6 * Subject to the GNU Public License, version 2. 7 * 8 * NUMA policy allows the user to give hints in which node(s) memory should 9 * be allocated. 10 * 11 * Support four policies per VMA and per process: 12 * 13 * The VMA policy has priority over the process policy for a page fault. 14 * 15 * interleave Allocate memory interleaved over a set of nodes, 16 * with normal fallback if it fails. 17 * For VMA based allocations this interleaves based on the 18 * offset into the backing object or offset into the mapping 19 * for anonymous memory. For process policy an process counter 20 * is used. 21 * 22 * bind Only allocate memory on a specific set of nodes, 23 * no fallback. 24 * FIXME: memory is allocated starting with the first node 25 * to the last. It would be better if bind would truly restrict 26 * the allocation to memory nodes instead 27 * 28 * preferred Try a specific node first before normal fallback. 29 * As a special case node -1 here means do the allocation 30 * on the local CPU. This is normally identical to default, 31 * but useful to set in a VMA when you have a non default 32 * process policy. 33 * 34 * default Allocate on the local node first, or when on a VMA 35 * use the process policy. This is what Linux always did 36 * in a NUMA aware kernel and still does by, ahem, default. 37 * 38 * The process policy is applied for most non interrupt memory allocations 39 * in that process' context. Interrupts ignore the policies and always 40 * try to allocate on the local CPU. The VMA policy is only applied for memory 41 * allocations for a VMA in the VM. 42 * 43 * Currently there are a few corner cases in swapping where the policy 44 * is not applied, but the majority should be handled. When process policy 45 * is used it is not remembered over swap outs/swap ins. 46 * 47 * Only the highest zone in the zone hierarchy gets policied. Allocations 48 * requesting a lower zone just use default policy. This implies that 49 * on systems with highmem kernel lowmem allocation don't get policied. 50 * Same with GFP_DMA allocations. 51 * 52 * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between 53 * all users and remembered even when nobody has memory mapped. 54 */ 55 56/* Notebook: 57 fix mmap readahead to honour policy and enable policy for any page cache 58 object 59 statistics for bigpages 60 global policy for page cache? currently it uses process policy. Requires 61 first item above. 62 handle mremap for shared memory (currently ignored for the policy) 63 grows down? 64 make bind policy root only? It can trigger oom much faster and the 65 kernel is not always grateful with that. 66*/ 67 68#include <linux/mempolicy.h> 69#include <linux/mm.h> 70#include <linux/highmem.h> 71#include <linux/hugetlb.h> 72#include <linux/kernel.h> 73#include <linux/sched.h> 74#include <linux/nodemask.h> 75#include <linux/cpuset.h> 76#include <linux/slab.h> 77#include <linux/string.h> 78#include <linux/module.h> 79#include <linux/nsproxy.h> 80#include <linux/interrupt.h> 81#include <linux/init.h> 82#include <linux/compat.h> 83#include <linux/swap.h> 84#include <linux/seq_file.h> 85#include <linux/proc_fs.h> 86#include <linux/migrate.h> 87#include <linux/ksm.h> 88#include <linux/rmap.h> 89#include <linux/security.h> 90#include <linux/syscalls.h> 91#include <linux/ctype.h> 92#include <linux/mm_inline.h> 93 94#include <asm/tlbflush.h> 95#include <asm/uaccess.h> 96 97#include "internal.h" 98 99/* Internal flags */ 100#define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */ 101#define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */ 102#define MPOL_MF_STATS (MPOL_MF_INTERNAL << 2) /* Gather statistics */ 103 104static struct kmem_cache *policy_cache; 105static struct kmem_cache *sn_cache; 106 107/* Highest zone. An specific allocation for a zone below that is not 108 policied. */ 109enum zone_type policy_zone = 0; 110 111/* 112 * run-time system-wide default policy => local allocation 113 */ 114struct mempolicy default_policy = { 115 .refcnt = ATOMIC_INIT(1), /* never free it */ 116 .mode = MPOL_PREFERRED, 117 .flags = MPOL_F_LOCAL, 118}; 119 120static const struct mempolicy_operations { 121 int (*create)(struct mempolicy *pol, const nodemask_t *nodes); 122 /* 123 * If read-side task has no lock to protect task->mempolicy, write-side 124 * task will rebind the task->mempolicy by two step. The first step is 125 * setting all the newly nodes, and the second step is cleaning all the 126 * disallowed nodes. In this way, we can avoid finding no node to alloc 127 * page. 128 * If we have a lock to protect task->mempolicy in read-side, we do 129 * rebind directly. 130 * 131 * step: 132 * MPOL_REBIND_ONCE - do rebind work at once 133 * MPOL_REBIND_STEP1 - set all the newly nodes 134 * MPOL_REBIND_STEP2 - clean all the disallowed nodes 135 */ 136 void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes, 137 enum mpol_rebind_step step); 138} mpol_ops[MPOL_MAX]; 139 140/* Check that the nodemask contains at least one populated zone */ 141static int is_valid_nodemask(const nodemask_t *nodemask) 142{ 143 int nd, k; 144 145 for_each_node_mask(nd, *nodemask) { 146 struct zone *z; 147 148 for (k = 0; k <= policy_zone; k++) { 149 z = &NODE_DATA(nd)->node_zones[k]; 150 if (z->present_pages > 0) 151 return 1; 152 } 153 } 154 155 return 0; 156} 157 158static inline int mpol_store_user_nodemask(const struct mempolicy *pol) 159{ 160 return pol->flags & MPOL_MODE_FLAGS; 161} 162 163static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig, 164 const nodemask_t *rel) 165{ 166 nodemask_t tmp; 167 nodes_fold(tmp, *orig, nodes_weight(*rel)); 168 nodes_onto(*ret, tmp, *rel); 169} 170 171static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes) 172{ 173 if (nodes_empty(*nodes)) 174 return -EINVAL; 175 pol->v.nodes = *nodes; 176 return 0; 177} 178 179static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes) 180{ 181 if (!nodes) 182 pol->flags |= MPOL_F_LOCAL; /* local allocation */ 183 else if (nodes_empty(*nodes)) 184 return -EINVAL; /* no allowed nodes */ 185 else 186 pol->v.preferred_node = first_node(*nodes); 187 return 0; 188} 189 190static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes) 191{ 192 if (!is_valid_nodemask(nodes)) 193 return -EINVAL; 194 pol->v.nodes = *nodes; 195 return 0; 196} 197 198/* 199 * mpol_set_nodemask is called after mpol_new() to set up the nodemask, if 200 * any, for the new policy. mpol_new() has already validated the nodes 201 * parameter with respect to the policy mode and flags. But, we need to 202 * handle an empty nodemask with MPOL_PREFERRED here. 203 * 204 * Must be called holding task's alloc_lock to protect task's mems_allowed 205 * and mempolicy. May also be called holding the mmap_semaphore for write. 206 */ 207static int mpol_set_nodemask(struct mempolicy *pol, 208 const nodemask_t *nodes, struct nodemask_scratch *nsc) 209{ 210 int ret; 211 212 /* if mode is MPOL_DEFAULT, pol is NULL. This is right. */ 213 if (pol == NULL) 214 return 0; 215 /* Check N_HIGH_MEMORY */ 216 nodes_and(nsc->mask1, 217 cpuset_current_mems_allowed, node_states[N_HIGH_MEMORY]); 218 219 VM_BUG_ON(!nodes); 220 if (pol->mode == MPOL_PREFERRED && nodes_empty(*nodes)) 221 nodes = NULL; /* explicit local allocation */ 222 else { 223 if (pol->flags & MPOL_F_RELATIVE_NODES) 224 mpol_relative_nodemask(&nsc->mask2, nodes,&nsc->mask1); 225 else 226 nodes_and(nsc->mask2, *nodes, nsc->mask1); 227 228 if (mpol_store_user_nodemask(pol)) 229 pol->w.user_nodemask = *nodes; 230 else 231 pol->w.cpuset_mems_allowed = 232 cpuset_current_mems_allowed; 233 } 234 235 if (nodes) 236 ret = mpol_ops[pol->mode].create(pol, &nsc->mask2); 237 else 238 ret = mpol_ops[pol->mode].create(pol, NULL); 239 return ret; 240} 241 242/* 243 * This function just creates a new policy, does some check and simple 244 * initialization. You must invoke mpol_set_nodemask() to set nodes. 245 */ 246static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags, 247 nodemask_t *nodes) 248{ 249 struct mempolicy *policy; 250 251 pr_debug("setting mode %d flags %d nodes[0] %lx\n", 252 mode, flags, nodes ? nodes_addr(*nodes)[0] : -1); 253 254 if (mode == MPOL_DEFAULT) { 255 if (nodes && !nodes_empty(*nodes)) 256 return ERR_PTR(-EINVAL); 257 return NULL; /* simply delete any existing policy */ 258 } 259 VM_BUG_ON(!nodes); 260 261 /* 262 * MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or 263 * MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation). 264 * All other modes require a valid pointer to a non-empty nodemask. 265 */ 266 if (mode == MPOL_PREFERRED) { 267 if (nodes_empty(*nodes)) { 268 if (((flags & MPOL_F_STATIC_NODES) || 269 (flags & MPOL_F_RELATIVE_NODES))) 270 return ERR_PTR(-EINVAL); 271 } 272 } else if (nodes_empty(*nodes)) 273 return ERR_PTR(-EINVAL); 274 policy = kmem_cache_alloc(policy_cache, GFP_KERNEL); 275 if (!policy) 276 return ERR_PTR(-ENOMEM); 277 atomic_set(&policy->refcnt, 1); 278 policy->mode = mode; 279 policy->flags = flags; 280 281 return policy; 282} 283 284/* Slow path of a mpol destructor. */ 285void __mpol_put(struct mempolicy *p) 286{ 287 if (!atomic_dec_and_test(&p->refcnt)) 288 return; 289 kmem_cache_free(policy_cache, p); 290} 291 292static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes, 293 enum mpol_rebind_step step) 294{ 295} 296 297/* 298 * step: 299 * MPOL_REBIND_ONCE - do rebind work at once 300 * MPOL_REBIND_STEP1 - set all the newly nodes 301 * MPOL_REBIND_STEP2 - clean all the disallowed nodes 302 */ 303static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes, 304 enum mpol_rebind_step step) 305{ 306 nodemask_t tmp; 307 308 if (pol->flags & MPOL_F_STATIC_NODES) 309 nodes_and(tmp, pol->w.user_nodemask, *nodes); 310 else if (pol->flags & MPOL_F_RELATIVE_NODES) 311 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes); 312 else { 313 /* 314 * if step == 1, we use ->w.cpuset_mems_allowed to cache the 315 * result 316 */ 317 if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP1) { 318 nodes_remap(tmp, pol->v.nodes, 319 pol->w.cpuset_mems_allowed, *nodes); 320 pol->w.cpuset_mems_allowed = step ? tmp : *nodes; 321 } else if (step == MPOL_REBIND_STEP2) { 322 tmp = pol->w.cpuset_mems_allowed; 323 pol->w.cpuset_mems_allowed = *nodes; 324 } else 325 BUG(); 326 } 327 328 if (nodes_empty(tmp)) 329 tmp = *nodes; 330 331 if (step == MPOL_REBIND_STEP1) 332 nodes_or(pol->v.nodes, pol->v.nodes, tmp); 333 else if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP2) 334 pol->v.nodes = tmp; 335 else 336 BUG(); 337 338 if (!node_isset(current->il_next, tmp)) { 339 current->il_next = next_node(current->il_next, tmp); 340 if (current->il_next >= MAX_NUMNODES) 341 current->il_next = first_node(tmp); 342 if (current->il_next >= MAX_NUMNODES) 343 current->il_next = numa_node_id(); 344 } 345} 346 347static void mpol_rebind_preferred(struct mempolicy *pol, 348 const nodemask_t *nodes, 349 enum mpol_rebind_step step) 350{ 351 nodemask_t tmp; 352 353 if (pol->flags & MPOL_F_STATIC_NODES) { 354 int node = first_node(pol->w.user_nodemask); 355 356 if (node_isset(node, *nodes)) { 357 pol->v.preferred_node = node; 358 pol->flags &= ~MPOL_F_LOCAL; 359 } else 360 pol->flags |= MPOL_F_LOCAL; 361 } else if (pol->flags & MPOL_F_RELATIVE_NODES) { 362 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes); 363 pol->v.preferred_node = first_node(tmp); 364 } else if (!(pol->flags & MPOL_F_LOCAL)) { 365 pol->v.preferred_node = node_remap(pol->v.preferred_node, 366 pol->w.cpuset_mems_allowed, 367 *nodes); 368 pol->w.cpuset_mems_allowed = *nodes; 369 } 370} 371 372/* 373 * mpol_rebind_policy - Migrate a policy to a different set of nodes 374 * 375 * If read-side task has no lock to protect task->mempolicy, write-side 376 * task will rebind the task->mempolicy by two step. The first step is 377 * setting all the newly nodes, and the second step is cleaning all the 378 * disallowed nodes. In this way, we can avoid finding no node to alloc 379 * page. 380 * If we have a lock to protect task->mempolicy in read-side, we do 381 * rebind directly. 382 * 383 * step: 384 * MPOL_REBIND_ONCE - do rebind work at once 385 * MPOL_REBIND_STEP1 - set all the newly nodes 386 * MPOL_REBIND_STEP2 - clean all the disallowed nodes 387 */ 388static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask, 389 enum mpol_rebind_step step) 390{ 391 if (!pol) 392 return; 393 if (!mpol_store_user_nodemask(pol) && step == 0 && 394 nodes_equal(pol->w.cpuset_mems_allowed, *newmask)) 395 return; 396 397 if (step == MPOL_REBIND_STEP1 && (pol->flags & MPOL_F_REBINDING)) 398 return; 399 400 if (step == MPOL_REBIND_STEP2 && !(pol->flags & MPOL_F_REBINDING)) 401 BUG(); 402 403 if (step == MPOL_REBIND_STEP1) 404 pol->flags |= MPOL_F_REBINDING; 405 else if (step == MPOL_REBIND_STEP2) 406 pol->flags &= ~MPOL_F_REBINDING; 407 else if (step >= MPOL_REBIND_NSTEP) 408 BUG(); 409 410 mpol_ops[pol->mode].rebind(pol, newmask, step); 411} 412 413/* 414 * Wrapper for mpol_rebind_policy() that just requires task 415 * pointer, and updates task mempolicy. 416 * 417 * Called with task's alloc_lock held. 418 */ 419 420void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new, 421 enum mpol_rebind_step step) 422{ 423 mpol_rebind_policy(tsk->mempolicy, new, step); 424} 425 426/* 427 * Rebind each vma in mm to new nodemask. 428 * 429 * Call holding a reference to mm. Takes mm->mmap_sem during call. 430 */ 431 432void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new) 433{ 434 struct vm_area_struct *vma; 435 436 down_write(&mm->mmap_sem); 437 for (vma = mm->mmap; vma; vma = vma->vm_next) 438 mpol_rebind_policy(vma->vm_policy, new, MPOL_REBIND_ONCE); 439 up_write(&mm->mmap_sem); 440} 441 442static const struct mempolicy_operations mpol_ops[MPOL_MAX] = { 443 [MPOL_DEFAULT] = { 444 .rebind = mpol_rebind_default, 445 }, 446 [MPOL_INTERLEAVE] = { 447 .create = mpol_new_interleave, 448 .rebind = mpol_rebind_nodemask, 449 }, 450 [MPOL_PREFERRED] = { 451 .create = mpol_new_preferred, 452 .rebind = mpol_rebind_preferred, 453 }, 454 [MPOL_BIND] = { 455 .create = mpol_new_bind, 456 .rebind = mpol_rebind_nodemask, 457 }, 458}; 459 460static void gather_stats(struct page *, void *, int pte_dirty); 461static void migrate_page_add(struct page *page, struct list_head *pagelist, 462 unsigned long flags); 463 464/* Scan through pages checking if pages follow certain conditions. */ 465static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd, 466 unsigned long addr, unsigned long end, 467 const nodemask_t *nodes, unsigned long flags, 468 void *private) 469{ 470 pte_t *orig_pte; 471 pte_t *pte; 472 spinlock_t *ptl; 473 474 orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 475 do { 476 struct page *page; 477 int nid; 478 479 if (!pte_present(*pte)) 480 continue; 481 page = vm_normal_page(vma, addr, *pte); 482 if (!page) 483 continue; 484 /* 485 * vm_normal_page() filters out zero pages, but there might 486 * still be PageReserved pages to skip, perhaps in a VDSO. 487 * And we cannot move PageKsm pages sensibly or safely yet. 488 */ 489 if (PageReserved(page) || PageKsm(page)) 490 continue; 491 nid = page_to_nid(page); 492 if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT)) 493 continue; 494 495 if (flags & MPOL_MF_STATS) 496 gather_stats(page, private, pte_dirty(*pte)); 497 else if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) 498 migrate_page_add(page, private, flags); 499 else 500 break; 501 } while (pte++, addr += PAGE_SIZE, addr != end); 502 pte_unmap_unlock(orig_pte, ptl); 503 return addr != end; 504} 505 506static inline int check_pmd_range(struct vm_area_struct *vma, pud_t *pud, 507 unsigned long addr, unsigned long end, 508 const nodemask_t *nodes, unsigned long flags, 509 void *private) 510{ 511 pmd_t *pmd; 512 unsigned long next; 513 514 pmd = pmd_offset(pud, addr); 515 do { 516 next = pmd_addr_end(addr, end); 517 if (pmd_none_or_clear_bad(pmd)) 518 continue; 519 if (check_pte_range(vma, pmd, addr, next, nodes, 520 flags, private)) 521 return -EIO; 522 } while (pmd++, addr = next, addr != end); 523 return 0; 524} 525 526static inline int check_pud_range(struct vm_area_struct *vma, pgd_t *pgd, 527 unsigned long addr, unsigned long end, 528 const nodemask_t *nodes, unsigned long flags, 529 void *private) 530{ 531 pud_t *pud; 532 unsigned long next; 533 534 pud = pud_offset(pgd, addr); 535 do { 536 next = pud_addr_end(addr, end); 537 if (pud_none_or_clear_bad(pud)) 538 continue; 539 if (check_pmd_range(vma, pud, addr, next, nodes, 540 flags, private)) 541 return -EIO; 542 } while (pud++, addr = next, addr != end); 543 return 0; 544} 545 546static inline int check_pgd_range(struct vm_area_struct *vma, 547 unsigned long addr, unsigned long end, 548 const nodemask_t *nodes, unsigned long flags, 549 void *private) 550{ 551 pgd_t *pgd; 552 unsigned long next; 553 554 pgd = pgd_offset(vma->vm_mm, addr); 555 do { 556 next = pgd_addr_end(addr, end); 557 if (pgd_none_or_clear_bad(pgd)) 558 continue; 559 if (check_pud_range(vma, pgd, addr, next, nodes, 560 flags, private)) 561 return -EIO; 562 } while (pgd++, addr = next, addr != end); 563 return 0; 564} 565 566/* 567 * Check if all pages in a range are on a set of nodes. 568 * If pagelist != NULL then isolate pages from the LRU and 569 * put them on the pagelist. 570 */ 571static struct vm_area_struct * 572check_range(struct mm_struct *mm, unsigned long start, unsigned long end, 573 const nodemask_t *nodes, unsigned long flags, void *private) 574{ 575 int err; 576 struct vm_area_struct *first, *vma, *prev; 577 578 579 first = find_vma(mm, start); 580 if (!first) 581 return ERR_PTR(-EFAULT); 582 prev = NULL; 583 for (vma = first; vma && vma->vm_start < end; vma = vma->vm_next) { 584 if (!(flags & MPOL_MF_DISCONTIG_OK)) { 585 if (!vma->vm_next && vma->vm_end < end) 586 return ERR_PTR(-EFAULT); 587 if (prev && prev->vm_end < vma->vm_start) 588 return ERR_PTR(-EFAULT); 589 } 590 if (!is_vm_hugetlb_page(vma) && 591 ((flags & MPOL_MF_STRICT) || 592 ((flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) && 593 vma_migratable(vma)))) { 594 unsigned long endvma = vma->vm_end; 595 596 if (endvma > end) 597 endvma = end; 598 if (vma->vm_start > start) 599 start = vma->vm_start; 600 err = check_pgd_range(vma, start, endvma, nodes, 601 flags, private); 602 if (err) { 603 first = ERR_PTR(err); 604 break; 605 } 606 } 607 prev = vma; 608 } 609 return first; 610} 611 612/* Apply policy to a single VMA */ 613static int policy_vma(struct vm_area_struct *vma, struct mempolicy *new) 614{ 615 int err = 0; 616 struct mempolicy *old = vma->vm_policy; 617 618 pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n", 619 vma->vm_start, vma->vm_end, vma->vm_pgoff, 620 vma->vm_ops, vma->vm_file, 621 vma->vm_ops ? vma->vm_ops->set_policy : NULL); 622 623 if (vma->vm_ops && vma->vm_ops->set_policy) 624 err = vma->vm_ops->set_policy(vma, new); 625 if (!err) { 626 mpol_get(new); 627 vma->vm_policy = new; 628 mpol_put(old); 629 } 630 return err; 631} 632 633/* Step 2: apply policy to a range and do splits. */ 634static int mbind_range(struct mm_struct *mm, unsigned long start, 635 unsigned long end, struct mempolicy *new_pol) 636{ 637 struct vm_area_struct *next; 638 struct vm_area_struct *prev; 639 struct vm_area_struct *vma; 640 int err = 0; 641 pgoff_t pgoff; 642 unsigned long vmstart; 643 unsigned long vmend; 644 645 vma = find_vma_prev(mm, start, &prev); 646 if (!vma || vma->vm_start > start) 647 return -EFAULT; 648 649 for (; vma && vma->vm_start < end; prev = vma, vma = next) { 650 next = vma->vm_next; 651 vmstart = max(start, vma->vm_start); 652 vmend = min(end, vma->vm_end); 653 654 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); 655 prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags, 656 vma->anon_vma, vma->vm_file, pgoff, new_pol); 657 if (prev) { 658 vma = prev; 659 next = vma->vm_next; 660 continue; 661 } 662 if (vma->vm_start != vmstart) { 663 err = split_vma(vma->vm_mm, vma, vmstart, 1); 664 if (err) 665 goto out; 666 } 667 if (vma->vm_end != vmend) { 668 err = split_vma(vma->vm_mm, vma, vmend, 0); 669 if (err) 670 goto out; 671 } 672 err = policy_vma(vma, new_pol); 673 if (err) 674 goto out; 675 } 676 677 out: 678 return err; 679} 680 681/* 682 * Update task->flags PF_MEMPOLICY bit: set iff non-default 683 * mempolicy. Allows more rapid checking of this (combined perhaps 684 * with other PF_* flag bits) on memory allocation hot code paths. 685 * 686 * If called from outside this file, the task 'p' should -only- be 687 * a newly forked child not yet visible on the task list, because 688 * manipulating the task flags of a visible task is not safe. 689 * 690 * The above limitation is why this routine has the funny name 691 * mpol_fix_fork_child_flag(). 692 * 693 * It is also safe to call this with a task pointer of current, 694 * which the static wrapper mpol_set_task_struct_flag() does, 695 * for use within this file. 696 */ 697 698void mpol_fix_fork_child_flag(struct task_struct *p) 699{ 700 if (p->mempolicy) 701 p->flags |= PF_MEMPOLICY; 702 else 703 p->flags &= ~PF_MEMPOLICY; 704} 705 706static void mpol_set_task_struct_flag(void) 707{ 708 mpol_fix_fork_child_flag(current); 709} 710 711/* Set the process memory policy */ 712static long do_set_mempolicy(unsigned short mode, unsigned short flags, 713 nodemask_t *nodes) 714{ 715 struct mempolicy *new, *old; 716 struct mm_struct *mm = current->mm; 717 NODEMASK_SCRATCH(scratch); 718 int ret; 719 720 if (!scratch) 721 return -ENOMEM; 722 723 new = mpol_new(mode, flags, nodes); 724 if (IS_ERR(new)) { 725 ret = PTR_ERR(new); 726 goto out; 727 } 728 /* 729 * prevent changing our mempolicy while show_numa_maps() 730 * is using it. 731 * Note: do_set_mempolicy() can be called at init time 732 * with no 'mm'. 733 */ 734 if (mm) 735 down_write(&mm->mmap_sem); 736 task_lock(current); 737 ret = mpol_set_nodemask(new, nodes, scratch); 738 if (ret) { 739 task_unlock(current); 740 if (mm) 741 up_write(&mm->mmap_sem); 742 mpol_put(new); 743 goto out; 744 } 745 old = current->mempolicy; 746 current->mempolicy = new; 747 mpol_set_task_struct_flag(); 748 if (new && new->mode == MPOL_INTERLEAVE && 749 nodes_weight(new->v.nodes)) 750 current->il_next = first_node(new->v.nodes); 751 task_unlock(current); 752 if (mm) 753 up_write(&mm->mmap_sem); 754 755 mpol_put(old); 756 ret = 0; 757out: 758 NODEMASK_SCRATCH_FREE(scratch); 759 return ret; 760} 761 762/* 763 * Return nodemask for policy for get_mempolicy() query 764 * 765 * Called with task's alloc_lock held 766 */ 767static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes) 768{ 769 nodes_clear(*nodes); 770 if (p == &default_policy) 771 return; 772 773 switch (p->mode) { 774 case MPOL_BIND: 775 /* Fall through */ 776 case MPOL_INTERLEAVE: 777 *nodes = p->v.nodes; 778 break; 779 case MPOL_PREFERRED: 780 if (!(p->flags & MPOL_F_LOCAL)) 781 node_set(p->v.preferred_node, *nodes); 782 /* else return empty node mask for local allocation */ 783 break; 784 default: 785 BUG(); 786 } 787} 788 789static int lookup_node(struct mm_struct *mm, unsigned long addr) 790{ 791 struct page *p; 792 int err; 793 794 err = get_user_pages(current, mm, addr & PAGE_MASK, 1, 0, 0, &p, NULL); 795 if (err >= 0) { 796 err = page_to_nid(p); 797 put_page(p); 798 } 799 return err; 800} 801 802/* Retrieve NUMA policy */ 803static long do_get_mempolicy(int *policy, nodemask_t *nmask, 804 unsigned long addr, unsigned long flags) 805{ 806 int err; 807 struct mm_struct *mm = current->mm; 808 struct vm_area_struct *vma = NULL; 809 struct mempolicy *pol = current->mempolicy; 810 811 if (flags & 812 ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED)) 813 return -EINVAL; 814 815 if (flags & MPOL_F_MEMS_ALLOWED) { 816 if (flags & (MPOL_F_NODE|MPOL_F_ADDR)) 817 return -EINVAL; 818 *policy = 0; /* just so it's initialized */ 819 task_lock(current); 820 *nmask = cpuset_current_mems_allowed; 821 task_unlock(current); 822 return 0; 823 } 824 825 if (flags & MPOL_F_ADDR) { 826 /* 827 * Do NOT fall back to task policy if the 828 * vma/shared policy at addr is NULL. We 829 * want to return MPOL_DEFAULT in this case. 830 */ 831 down_read(&mm->mmap_sem); 832 vma = find_vma_intersection(mm, addr, addr+1); 833 if (!vma) { 834 up_read(&mm->mmap_sem); 835 return -EFAULT; 836 } 837 if (vma->vm_ops && vma->vm_ops->get_policy) 838 pol = vma->vm_ops->get_policy(vma, addr); 839 else 840 pol = vma->vm_policy; 841 } else if (addr) 842 return -EINVAL; 843 844 if (!pol) 845 pol = &default_policy; /* indicates default behavior */ 846 847 if (flags & MPOL_F_NODE) { 848 if (flags & MPOL_F_ADDR) { 849 err = lookup_node(mm, addr); 850 if (err < 0) 851 goto out; 852 *policy = err; 853 } else if (pol == current->mempolicy && 854 pol->mode == MPOL_INTERLEAVE) { 855 *policy = current->il_next; 856 } else { 857 err = -EINVAL; 858 goto out; 859 } 860 } else { 861 *policy = pol == &default_policy ? MPOL_DEFAULT : 862 pol->mode; 863 /* 864 * Internal mempolicy flags must be masked off before exposing 865 * the policy to userspace. 866 */ 867 *policy |= (pol->flags & MPOL_MODE_FLAGS); 868 } 869 870 if (vma) { 871 up_read(¤t->mm->mmap_sem); 872 vma = NULL; 873 } 874 875 err = 0; 876 if (nmask) { 877 if (mpol_store_user_nodemask(pol)) { 878 *nmask = pol->w.user_nodemask; 879 } else { 880 task_lock(current); 881 get_policy_nodemask(pol, nmask); 882 task_unlock(current); 883 } 884 } 885 886 out: 887 mpol_cond_put(pol); 888 if (vma) 889 up_read(¤t->mm->mmap_sem); 890 return err; 891} 892 893#ifdef CONFIG_MIGRATION 894/* 895 * page migration 896 */ 897static void migrate_page_add(struct page *page, struct list_head *pagelist, 898 unsigned long flags) 899{ 900 /* 901 * Avoid migrating a page that is shared with others. 902 */ 903 if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1) { 904 if (!isolate_lru_page(page)) { 905 list_add_tail(&page->lru, pagelist); 906 inc_zone_page_state(page, NR_ISOLATED_ANON + 907 page_is_file_cache(page)); 908 } 909 } 910} 911 912static struct page *new_node_page(struct page *page, unsigned long node, int **x) 913{ 914 return alloc_pages_exact_node(node, GFP_HIGHUSER_MOVABLE, 0); 915} 916 917/* 918 * Migrate pages from one node to a target node. 919 * Returns error or the number of pages not migrated. 920 */ 921static int migrate_to_node(struct mm_struct *mm, int source, int dest, 922 int flags) 923{ 924 nodemask_t nmask; 925 LIST_HEAD(pagelist); 926 int err = 0; 927 struct vm_area_struct *vma; 928 929 nodes_clear(nmask); 930 node_set(source, nmask); 931 932 vma = check_range(mm, mm->mmap->vm_start, mm->task_size, &nmask, 933 flags | MPOL_MF_DISCONTIG_OK, &pagelist); 934 if (IS_ERR(vma)) 935 return PTR_ERR(vma); 936 937 if (!list_empty(&pagelist)) { 938 err = migrate_pages(&pagelist, new_node_page, dest, 0); 939 if (err) 940 putback_lru_pages(&pagelist); 941 } 942 943 return err; 944} 945 946/* 947 * Move pages between the two nodesets so as to preserve the physical 948 * layout as much as possible. 949 * 950 * Returns the number of page that could not be moved. 951 */ 952int do_migrate_pages(struct mm_struct *mm, 953 const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags) 954{ 955 int busy = 0; 956 int err; 957 nodemask_t tmp; 958 959 err = migrate_prep(); 960 if (err) 961 return err; 962 963 down_read(&mm->mmap_sem); 964 965 err = migrate_vmas(mm, from_nodes, to_nodes, flags); 966 if (err) 967 goto out; 968 969 /* 970 * Find a 'source' bit set in 'tmp' whose corresponding 'dest' 971 * bit in 'to' is not also set in 'tmp'. Clear the found 'source' 972 * bit in 'tmp', and return that <source, dest> pair for migration. 973 * The pair of nodemasks 'to' and 'from' define the map. 974 * 975 * If no pair of bits is found that way, fallback to picking some 976 * pair of 'source' and 'dest' bits that are not the same. If the 977 * 'source' and 'dest' bits are the same, this represents a node 978 * that will be migrating to itself, so no pages need move. 979 * 980 * If no bits are left in 'tmp', or if all remaining bits left 981 * in 'tmp' correspond to the same bit in 'to', return false 982 * (nothing left to migrate). 983 * 984 * This lets us pick a pair of nodes to migrate between, such that 985 * if possible the dest node is not already occupied by some other 986 * source node, minimizing the risk of overloading the memory on a 987 * node that would happen if we migrated incoming memory to a node 988 * before migrating outgoing memory source that same node. 989 * 990 * A single scan of tmp is sufficient. As we go, we remember the 991 * most recent <s, d> pair that moved (s != d). If we find a pair 992 * that not only moved, but what's better, moved to an empty slot 993 * (d is not set in tmp), then we break out then, with that pair. 994 * Otherwise when we finish scannng from_tmp, we at least have the 995 * most recent <s, d> pair that moved. If we get all the way through 996 * the scan of tmp without finding any node that moved, much less 997 * moved to an empty node, then there is nothing left worth migrating. 998 */ 999 1000 tmp = *from_nodes; 1001 while (!nodes_empty(tmp)) { 1002 int s,d; 1003 int source = -1; 1004 int dest = 0; 1005 1006 for_each_node_mask(s, tmp) { 1007 d = node_remap(s, *from_nodes, *to_nodes); 1008 if (s == d) 1009 continue; 1010 1011 source = s; /* Node moved. Memorize */ 1012 dest = d; 1013 1014 /* dest not in remaining from nodes? */ 1015 if (!node_isset(dest, tmp)) 1016 break; 1017 } 1018 if (source == -1) 1019 break; 1020 1021 node_clear(source, tmp); 1022 err = migrate_to_node(mm, source, dest, flags); 1023 if (err > 0) 1024 busy += err; 1025 if (err < 0) 1026 break; 1027 } 1028out: 1029 up_read(&mm->mmap_sem); 1030 if (err < 0) 1031 return err; 1032 return busy; 1033 1034} 1035 1036/* 1037 * Allocate a new page for page migration based on vma policy. 1038 * Start assuming that page is mapped by vma pointed to by @private. 1039 * Search forward from there, if not. N.B., this assumes that the 1040 * list of pages handed to migrate_pages()--which is how we get here-- 1041 * is in virtual address order. 1042 */ 1043static struct page *new_vma_page(struct page *page, unsigned long private, int **x) 1044{ 1045 struct vm_area_struct *vma = (struct vm_area_struct *)private; 1046 unsigned long uninitialized_var(address); 1047 1048 while (vma) { 1049 address = page_address_in_vma(page, vma); 1050 if (address != -EFAULT) 1051 break; 1052 vma = vma->vm_next; 1053 } 1054 1055 /* 1056 * if !vma, alloc_page_vma() will use task or system default policy 1057 */ 1058 return alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address); 1059} 1060#else 1061 1062static void migrate_page_add(struct page *page, struct list_head *pagelist, 1063 unsigned long flags) 1064{ 1065} 1066 1067int do_migrate_pages(struct mm_struct *mm, 1068 const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags) 1069{ 1070 return -ENOSYS; 1071} 1072 1073static struct page *new_vma_page(struct page *page, unsigned long private, int **x) 1074{ 1075 return NULL; 1076} 1077#endif 1078 1079static long do_mbind(unsigned long start, unsigned long len, 1080 unsigned short mode, unsigned short mode_flags, 1081 nodemask_t *nmask, unsigned long flags) 1082{ 1083 struct vm_area_struct *vma; 1084 struct mm_struct *mm = current->mm; 1085 struct mempolicy *new; 1086 unsigned long end; 1087 int err; 1088 LIST_HEAD(pagelist); 1089 1090 if (flags & ~(unsigned long)(MPOL_MF_STRICT | 1091 MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) 1092 return -EINVAL; 1093 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE)) 1094 return -EPERM; 1095 1096 if (start & ~PAGE_MASK) 1097 return -EINVAL; 1098 1099 if (mode == MPOL_DEFAULT) 1100 flags &= ~MPOL_MF_STRICT; 1101 1102 len = (len + PAGE_SIZE - 1) & PAGE_MASK; 1103 end = start + len; 1104 1105 if (end < start) 1106 return -EINVAL; 1107 if (end == start) 1108 return 0; 1109 1110 new = mpol_new(mode, mode_flags, nmask); 1111 if (IS_ERR(new)) 1112 return PTR_ERR(new); 1113 1114 /* 1115 * If we are using the default policy then operation 1116 * on discontinuous address spaces is okay after all 1117 */ 1118 if (!new) 1119 flags |= MPOL_MF_DISCONTIG_OK; 1120 1121 pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n", 1122 start, start + len, mode, mode_flags, 1123 nmask ? nodes_addr(*nmask)[0] : -1); 1124 1125 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) { 1126 1127 err = migrate_prep(); 1128 if (err) 1129 goto mpol_out; 1130 } 1131 { 1132 NODEMASK_SCRATCH(scratch); 1133 if (scratch) { 1134 down_write(&mm->mmap_sem); 1135 task_lock(current); 1136 err = mpol_set_nodemask(new, nmask, scratch); 1137 task_unlock(current); 1138 if (err) 1139 up_write(&mm->mmap_sem); 1140 } else 1141 err = -ENOMEM; 1142 NODEMASK_SCRATCH_FREE(scratch); 1143 } 1144 if (err) 1145 goto mpol_out; 1146 1147 vma = check_range(mm, start, end, nmask, 1148 flags | MPOL_MF_INVERT, &pagelist); 1149 1150 err = PTR_ERR(vma); 1151 if (!IS_ERR(vma)) { 1152 int nr_failed = 0; 1153 1154 err = mbind_range(mm, start, end, new); 1155 1156 if (!list_empty(&pagelist)) { 1157 nr_failed = migrate_pages(&pagelist, new_vma_page, 1158 (unsigned long)vma, 0); 1159 if (nr_failed) 1160 putback_lru_pages(&pagelist); 1161 } 1162 1163 if (!err && nr_failed && (flags & MPOL_MF_STRICT)) 1164 err = -EIO; 1165 } else 1166 putback_lru_pages(&pagelist); 1167 1168 up_write(&mm->mmap_sem); 1169 mpol_out: 1170 mpol_put(new); 1171 return err; 1172} 1173 1174/* 1175 * User space interface with variable sized bitmaps for nodelists. 1176 */ 1177 1178/* Copy a node mask from user space. */ 1179static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask, 1180 unsigned long maxnode) 1181{ 1182 unsigned long k; 1183 unsigned long nlongs; 1184 unsigned long endmask; 1185 1186 --maxnode; 1187 nodes_clear(*nodes); 1188 if (maxnode == 0 || !nmask) 1189 return 0; 1190 if (maxnode > PAGE_SIZE*BITS_PER_BYTE) 1191 return -EINVAL; 1192 1193 nlongs = BITS_TO_LONGS(maxnode); 1194 if ((maxnode % BITS_PER_LONG) == 0) 1195 endmask = ~0UL; 1196 else 1197 endmask = (1UL << (maxnode % BITS_PER_LONG)) - 1; 1198 1199 /* When the user specified more nodes than supported just check 1200 if the non supported part is all zero. */ 1201 if (nlongs > BITS_TO_LONGS(MAX_NUMNODES)) { 1202 if (nlongs > PAGE_SIZE/sizeof(long)) 1203 return -EINVAL; 1204 for (k = BITS_TO_LONGS(MAX_NUMNODES); k < nlongs; k++) { 1205 unsigned long t; 1206 if (get_user(t, nmask + k)) 1207 return -EFAULT; 1208 if (k == nlongs - 1) { 1209 if (t & endmask) 1210 return -EINVAL; 1211 } else if (t) 1212 return -EINVAL; 1213 } 1214 nlongs = BITS_TO_LONGS(MAX_NUMNODES); 1215 endmask = ~0UL; 1216 } 1217 1218 if (copy_from_user(nodes_addr(*nodes), nmask, nlongs*sizeof(unsigned long))) 1219 return -EFAULT; 1220 nodes_addr(*nodes)[nlongs-1] &= endmask; 1221 return 0; 1222} 1223 1224/* Copy a kernel node mask to user space */ 1225static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode, 1226 nodemask_t *nodes) 1227{ 1228 unsigned long copy = ALIGN(maxnode-1, 64) / 8; 1229 const int nbytes = BITS_TO_LONGS(MAX_NUMNODES) * sizeof(long); 1230 1231 if (copy > nbytes) { 1232 if (copy > PAGE_SIZE) 1233 return -EINVAL; 1234 if (clear_user((char __user *)mask + nbytes, copy - nbytes)) 1235 return -EFAULT; 1236 copy = nbytes; 1237 } 1238 return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0; 1239} 1240 1241SYSCALL_DEFINE6(mbind, unsigned long, start, unsigned long, len, 1242 unsigned long, mode, unsigned long __user *, nmask, 1243 unsigned long, maxnode, unsigned, flags) 1244{ 1245 nodemask_t nodes; 1246 int err; 1247 unsigned short mode_flags; 1248 1249 mode_flags = mode & MPOL_MODE_FLAGS; 1250 mode &= ~MPOL_MODE_FLAGS; 1251 if (mode >= MPOL_MAX) 1252 return -EINVAL; 1253 if ((mode_flags & MPOL_F_STATIC_NODES) && 1254 (mode_flags & MPOL_F_RELATIVE_NODES)) 1255 return -EINVAL; 1256 err = get_nodes(&nodes, nmask, maxnode); 1257 if (err) 1258 return err; 1259 return do_mbind(start, len, mode, mode_flags, &nodes, flags); 1260} 1261 1262/* Set the process memory policy */ 1263SYSCALL_DEFINE3(set_mempolicy, int, mode, unsigned long __user *, nmask, 1264 unsigned long, maxnode) 1265{ 1266 int err; 1267 nodemask_t nodes; 1268 unsigned short flags; 1269 1270 flags = mode & MPOL_MODE_FLAGS; 1271 mode &= ~MPOL_MODE_FLAGS; 1272 if ((unsigned int)mode >= MPOL_MAX) 1273 return -EINVAL; 1274 if ((flags & MPOL_F_STATIC_NODES) && (flags & MPOL_F_RELATIVE_NODES)) 1275 return -EINVAL; 1276 err = get_nodes(&nodes, nmask, maxnode); 1277 if (err) 1278 return err; 1279 return do_set_mempolicy(mode, flags, &nodes); 1280} 1281 1282SYSCALL_DEFINE4(migrate_pages, pid_t, pid, unsigned long, maxnode, 1283 const unsigned long __user *, old_nodes, 1284 const unsigned long __user *, new_nodes) 1285{ 1286 const struct cred *cred = current_cred(), *tcred; 1287 struct mm_struct *mm = NULL; 1288 struct task_struct *task; 1289 nodemask_t task_nodes; 1290 int err; 1291 nodemask_t *old; 1292 nodemask_t *new; 1293 NODEMASK_SCRATCH(scratch); 1294 1295 if (!scratch) 1296 return -ENOMEM; 1297 1298 old = &scratch->mask1; 1299 new = &scratch->mask2; 1300 1301 err = get_nodes(old, old_nodes, maxnode); 1302 if (err) 1303 goto out; 1304 1305 err = get_nodes(new, new_nodes, maxnode); 1306 if (err) 1307 goto out; 1308 1309 /* Find the mm_struct */ 1310 read_lock(&tasklist_lock); 1311 task = pid ? find_task_by_vpid(pid) : current; 1312 if (!task) { 1313 read_unlock(&tasklist_lock); 1314 err = -ESRCH; 1315 goto out; 1316 } 1317 mm = get_task_mm(task); 1318 read_unlock(&tasklist_lock); 1319 1320 err = -EINVAL; 1321 if (!mm) 1322 goto out; 1323 1324 /* 1325 * Check if this process has the right to modify the specified 1326 * process. The right exists if the process has administrative 1327 * capabilities, superuser privileges or the same 1328 * userid as the target process. 1329 */ 1330 rcu_read_lock(); 1331 tcred = __task_cred(task); 1332 if (cred->euid != tcred->suid && cred->euid != tcred->uid && 1333 cred->uid != tcred->suid && cred->uid != tcred->uid && 1334 !capable(CAP_SYS_NICE)) { 1335 rcu_read_unlock(); 1336 err = -EPERM; 1337 goto out; 1338 } 1339 rcu_read_unlock(); 1340 1341 task_nodes = cpuset_mems_allowed(task); 1342 /* Is the user allowed to access the target nodes? */ 1343 if (!nodes_subset(*new, task_nodes) && !capable(CAP_SYS_NICE)) { 1344 err = -EPERM; 1345 goto out; 1346 } 1347 1348 if (!nodes_subset(*new, node_states[N_HIGH_MEMORY])) { 1349 err = -EINVAL; 1350 goto out; 1351 } 1352 1353 err = security_task_movememory(task); 1354 if (err) 1355 goto out; 1356 1357 err = do_migrate_pages(mm, old, new, 1358 capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE); 1359out: 1360 if (mm) 1361 mmput(mm); 1362 NODEMASK_SCRATCH_FREE(scratch); 1363 1364 return err; 1365} 1366 1367 1368/* Retrieve NUMA policy */ 1369SYSCALL_DEFINE5(get_mempolicy, int __user *, policy, 1370 unsigned long __user *, nmask, unsigned long, maxnode, 1371 unsigned long, addr, unsigned long, flags) 1372{ 1373 int err; 1374 int uninitialized_var(pval); 1375 nodemask_t nodes; 1376 1377 if (nmask != NULL && maxnode < MAX_NUMNODES) 1378 return -EINVAL; 1379 1380 err = do_get_mempolicy(&pval, &nodes, addr, flags); 1381 1382 if (err) 1383 return err; 1384 1385 if (policy && put_user(pval, policy)) 1386 return -EFAULT; 1387 1388 if (nmask) 1389 err = copy_nodes_to_user(nmask, maxnode, &nodes); 1390 1391 return err; 1392} 1393 1394#ifdef CONFIG_COMPAT 1395 1396asmlinkage long compat_sys_get_mempolicy(int __user *policy, 1397 compat_ulong_t __user *nmask, 1398 compat_ulong_t maxnode, 1399 compat_ulong_t addr, compat_ulong_t flags) 1400{ 1401 long err; 1402 unsigned long __user *nm = NULL; 1403 unsigned long nr_bits, alloc_size; 1404 DECLARE_BITMAP(bm, MAX_NUMNODES); 1405 1406 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES); 1407 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8; 1408 1409 if (nmask) 1410 nm = compat_alloc_user_space(alloc_size); 1411 1412 err = sys_get_mempolicy(policy, nm, nr_bits+1, addr, flags); 1413 1414 if (!err && nmask) { 1415 err = copy_from_user(bm, nm, alloc_size); 1416 /* ensure entire bitmap is zeroed */ 1417 err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8); 1418 err |= compat_put_bitmap(nmask, bm, nr_bits); 1419 } 1420 1421 return err; 1422} 1423 1424asmlinkage long compat_sys_set_mempolicy(int mode, compat_ulong_t __user *nmask, 1425 compat_ulong_t maxnode) 1426{ 1427 long err = 0; 1428 unsigned long __user *nm = NULL; 1429 unsigned long nr_bits, alloc_size; 1430 DECLARE_BITMAP(bm, MAX_NUMNODES); 1431 1432 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES); 1433 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8; 1434 1435 if (nmask) { 1436 err = compat_get_bitmap(bm, nmask, nr_bits); 1437 nm = compat_alloc_user_space(alloc_size); 1438 err |= copy_to_user(nm, bm, alloc_size); 1439 } 1440 1441 if (err) 1442 return -EFAULT; 1443 1444 return sys_set_mempolicy(mode, nm, nr_bits+1); 1445} 1446 1447asmlinkage long compat_sys_mbind(compat_ulong_t start, compat_ulong_t len, 1448 compat_ulong_t mode, compat_ulong_t __user *nmask, 1449 compat_ulong_t maxnode, compat_ulong_t flags) 1450{ 1451 long err = 0; 1452 unsigned long __user *nm = NULL; 1453 unsigned long nr_bits, alloc_size; 1454 nodemask_t bm; 1455 1456 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES); 1457 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8; 1458 1459 if (nmask) { 1460 err = compat_get_bitmap(nodes_addr(bm), nmask, nr_bits); 1461 nm = compat_alloc_user_space(alloc_size); 1462 err |= copy_to_user(nm, nodes_addr(bm), alloc_size); 1463 } 1464 1465 if (err) 1466 return -EFAULT; 1467 1468 return sys_mbind(start, len, mode, nm, nr_bits+1, flags); 1469} 1470 1471#endif 1472 1473/* 1474 * get_vma_policy(@task, @vma, @addr) 1475 * @task - task for fallback if vma policy == default 1476 * @vma - virtual memory area whose policy is sought 1477 * @addr - address in @vma for shared policy lookup 1478 * 1479 * Returns effective policy for a VMA at specified address. 1480 * Falls back to @task or system default policy, as necessary. 1481 * Current or other task's task mempolicy and non-shared vma policies 1482 * are protected by the task's mmap_sem, which must be held for read by 1483 * the caller. 1484 * Shared policies [those marked as MPOL_F_SHARED] require an extra reference 1485 * count--added by the get_policy() vm_op, as appropriate--to protect against 1486 * freeing by another task. It is the caller's responsibility to free the 1487 * extra reference for shared policies. 1488 */ 1489static struct mempolicy *get_vma_policy(struct task_struct *task, 1490 struct vm_area_struct *vma, unsigned long addr) 1491{ 1492 struct mempolicy *pol = task->mempolicy; 1493 1494 if (vma) { 1495 if (vma->vm_ops && vma->vm_ops->get_policy) { 1496 struct mempolicy *vpol = vma->vm_ops->get_policy(vma, 1497 addr); 1498 if (vpol) 1499 pol = vpol; 1500 } else if (vma->vm_policy) 1501 pol = vma->vm_policy; 1502 } 1503 if (!pol) 1504 pol = &default_policy; 1505 return pol; 1506} 1507 1508/* 1509 * Return a nodemask representing a mempolicy for filtering nodes for 1510 * page allocation 1511 */ 1512static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy) 1513{ 1514 /* Lower zones don't get a nodemask applied for MPOL_BIND */ 1515 if (unlikely(policy->mode == MPOL_BIND) && 1516 gfp_zone(gfp) >= policy_zone && 1517 cpuset_nodemask_valid_mems_allowed(&policy->v.nodes)) 1518 return &policy->v.nodes; 1519 1520 return NULL; 1521} 1522 1523/* Return a zonelist indicated by gfp for node representing a mempolicy */ 1524static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy) 1525{ 1526 int nd = numa_node_id(); 1527 1528 switch (policy->mode) { 1529 case MPOL_PREFERRED: 1530 if (!(policy->flags & MPOL_F_LOCAL)) 1531 nd = policy->v.preferred_node; 1532 break; 1533 case MPOL_BIND: 1534 /* 1535 * Normally, MPOL_BIND allocations are node-local within the 1536 * allowed nodemask. However, if __GFP_THISNODE is set and the 1537 * current node isn't part of the mask, we use the zonelist for 1538 * the first node in the mask instead. 1539 */ 1540 if (unlikely(gfp & __GFP_THISNODE) && 1541 unlikely(!node_isset(nd, policy->v.nodes))) 1542 nd = first_node(policy->v.nodes); 1543 break; 1544 default: 1545 BUG(); 1546 } 1547 return node_zonelist(nd, gfp); 1548} 1549 1550/* Do dynamic interleaving for a process */ 1551static unsigned interleave_nodes(struct mempolicy *policy) 1552{ 1553 unsigned nid, next; 1554 struct task_struct *me = current; 1555 1556 nid = me->il_next; 1557 next = next_node(nid, policy->v.nodes); 1558 if (next >= MAX_NUMNODES) 1559 next = first_node(policy->v.nodes); 1560 if (next < MAX_NUMNODES) 1561 me->il_next = next; 1562 return nid; 1563} 1564 1565/* 1566 * Depending on the memory policy provide a node from which to allocate the 1567 * next slab entry. 1568 * @policy must be protected by freeing by the caller. If @policy is 1569 * the current task's mempolicy, this protection is implicit, as only the 1570 * task can change it's policy. The system default policy requires no 1571 * such protection. 1572 */ 1573unsigned slab_node(struct mempolicy *policy) 1574{ 1575 if (!policy || policy->flags & MPOL_F_LOCAL) 1576 return numa_node_id(); 1577 1578 switch (policy->mode) { 1579 case MPOL_PREFERRED: 1580 /* 1581 * handled MPOL_F_LOCAL above 1582 */ 1583 return policy->v.preferred_node; 1584 1585 case MPOL_INTERLEAVE: 1586 return interleave_nodes(policy); 1587 1588 case MPOL_BIND: { 1589 /* 1590 * Follow bind policy behavior and start allocation at the 1591 * first node. 1592 */ 1593 struct zonelist *zonelist; 1594 struct zone *zone; 1595 enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL); 1596 zonelist = &NODE_DATA(numa_node_id())->node_zonelists[0]; 1597 (void)first_zones_zonelist(zonelist, highest_zoneidx, 1598 &policy->v.nodes, 1599 &zone); 1600 return zone->node; 1601 } 1602 1603 default: 1604 BUG(); 1605 } 1606} 1607 1608/* Do static interleaving for a VMA with known offset. */ 1609static unsigned offset_il_node(struct mempolicy *pol, 1610 struct vm_area_struct *vma, unsigned long off) 1611{ 1612 unsigned nnodes = nodes_weight(pol->v.nodes); 1613 unsigned target; 1614 int c; 1615 int nid = -1; 1616 1617 if (!nnodes) 1618 return numa_node_id(); 1619 target = (unsigned int)off % nnodes; 1620 c = 0; 1621 do { 1622 nid = next_node(nid, pol->v.nodes); 1623 c++; 1624 } while (c <= target); 1625 return nid; 1626} 1627 1628/* Determine a node number for interleave */ 1629static inline unsigned interleave_nid(struct mempolicy *pol, 1630 struct vm_area_struct *vma, unsigned long addr, int shift) 1631{ 1632 if (vma) { 1633 unsigned long off; 1634 1635 /* 1636 * for small pages, there is no difference between 1637 * shift and PAGE_SHIFT, so the bit-shift is safe. 1638 * for huge pages, since vm_pgoff is in units of small 1639 * pages, we need to shift off the always 0 bits to get 1640 * a useful offset. 1641 */ 1642 BUG_ON(shift < PAGE_SHIFT); 1643 off = vma->vm_pgoff >> (shift - PAGE_SHIFT); 1644 off += (addr - vma->vm_start) >> shift; 1645 return offset_il_node(pol, vma, off); 1646 } else 1647 return interleave_nodes(pol); 1648} 1649 1650#ifdef CONFIG_HUGETLBFS 1651/* 1652 * huge_zonelist(@vma, @addr, @gfp_flags, @mpol) 1653 * @vma = virtual memory area whose policy is sought 1654 * @addr = address in @vma for shared policy lookup and interleave policy 1655 * @gfp_flags = for requested zone 1656 * @mpol = pointer to mempolicy pointer for reference counted mempolicy 1657 * @nodemask = pointer to nodemask pointer for MPOL_BIND nodemask 1658 * 1659 * Returns a zonelist suitable for a huge page allocation and a pointer 1660 * to the struct mempolicy for conditional unref after allocation. 1661 * If the effective policy is 'BIND, returns a pointer to the mempolicy's 1662 * @nodemask for filtering the zonelist. 1663 * 1664 * Must be protected by get_mems_allowed() 1665 */ 1666struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr, 1667 gfp_t gfp_flags, struct mempolicy **mpol, 1668 nodemask_t **nodemask) 1669{ 1670 struct zonelist *zl; 1671 1672 *mpol = get_vma_policy(current, vma, addr); 1673 *nodemask = NULL; /* assume !MPOL_BIND */ 1674 1675 if (unlikely((*mpol)->mode == MPOL_INTERLEAVE)) { 1676 zl = node_zonelist(interleave_nid(*mpol, vma, addr, 1677 huge_page_shift(hstate_vma(vma))), gfp_flags); 1678 } else { 1679 zl = policy_zonelist(gfp_flags, *mpol); 1680 if ((*mpol)->mode == MPOL_BIND) 1681 *nodemask = &(*mpol)->v.nodes; 1682 } 1683 return zl; 1684} 1685 1686/* 1687 * init_nodemask_of_mempolicy 1688 * 1689 * If the current task's mempolicy is "default" [NULL], return 'false' 1690 * to indicate default policy. Otherwise, extract the policy nodemask 1691 * for 'bind' or 'interleave' policy into the argument nodemask, or 1692 * initialize the argument nodemask to contain the single node for 1693 * 'preferred' or 'local' policy and return 'true' to indicate presence 1694 * of non-default mempolicy. 1695 * 1696 * We don't bother with reference counting the mempolicy [mpol_get/put] 1697 * because the current task is examining it's own mempolicy and a task's 1698 * mempolicy is only ever changed by the task itself. 1699 * 1700 * N.B., it is the caller's responsibility to free a returned nodemask. 1701 */ 1702bool init_nodemask_of_mempolicy(nodemask_t *mask) 1703{ 1704 struct mempolicy *mempolicy; 1705 int nid; 1706 1707 if (!(mask && current->mempolicy)) 1708 return false; 1709 1710 task_lock(current); 1711 mempolicy = current->mempolicy; 1712 switch (mempolicy->mode) { 1713 case MPOL_PREFERRED: 1714 if (mempolicy->flags & MPOL_F_LOCAL) 1715 nid = numa_node_id(); 1716 else 1717 nid = mempolicy->v.preferred_node; 1718 init_nodemask_of_node(mask, nid); 1719 break; 1720 1721 case MPOL_BIND: 1722 /* Fall through */ 1723 case MPOL_INTERLEAVE: 1724 *mask = mempolicy->v.nodes; 1725 break; 1726 1727 default: 1728 BUG(); 1729 } 1730 task_unlock(current); 1731 1732 return true; 1733} 1734#endif 1735 1736/* 1737 * mempolicy_nodemask_intersects 1738 * 1739 * If tsk's mempolicy is "default" [NULL], return 'true' to indicate default 1740 * policy. Otherwise, check for intersection between mask and the policy 1741 * nodemask for 'bind' or 'interleave' policy. For 'perferred' or 'local' 1742 * policy, always return true since it may allocate elsewhere on fallback. 1743 * 1744 * Takes task_lock(tsk) to prevent freeing of its mempolicy. 1745 */ 1746bool mempolicy_nodemask_intersects(struct task_struct *tsk, 1747 const nodemask_t *mask) 1748{ 1749 struct mempolicy *mempolicy; 1750 bool ret = true; 1751 1752 if (!mask) 1753 return ret; 1754 task_lock(tsk); 1755 mempolicy = tsk->mempolicy; 1756 if (!mempolicy) 1757 goto out; 1758 1759 switch (mempolicy->mode) { 1760 case MPOL_PREFERRED: 1761 /* 1762 * MPOL_PREFERRED and MPOL_F_LOCAL are only preferred nodes to 1763 * allocate from, they may fallback to other nodes when oom. 1764 * Thus, it's possible for tsk to have allocated memory from 1765 * nodes in mask. 1766 */ 1767 break; 1768 case MPOL_BIND: 1769 case MPOL_INTERLEAVE: 1770 ret = nodes_intersects(mempolicy->v.nodes, *mask); 1771 break; 1772 default: 1773 BUG(); 1774 } 1775out: 1776 task_unlock(tsk); 1777 return ret; 1778} 1779 1780/* Allocate a page in interleaved policy. 1781 Own path because it needs to do special accounting. */ 1782static struct page *alloc_page_interleave(gfp_t gfp, unsigned order, 1783 unsigned nid) 1784{ 1785 struct zonelist *zl; 1786 struct page *page; 1787 1788 zl = node_zonelist(nid, gfp); 1789 page = __alloc_pages(gfp, order, zl); 1790 if (page && page_zone(page) == zonelist_zone(&zl->_zonerefs[0])) 1791 inc_zone_page_state(page, NUMA_INTERLEAVE_HIT); 1792 return page; 1793} 1794 1795/** 1796 * alloc_page_vma - Allocate a page for a VMA. 1797 * 1798 * @gfp: 1799 * %GFP_USER user allocation. 1800 * %GFP_KERNEL kernel allocations, 1801 * %GFP_HIGHMEM highmem/user allocations, 1802 * %GFP_FS allocation should not call back into a file system. 1803 * %GFP_ATOMIC don't sleep. 1804 * 1805 * @vma: Pointer to VMA or NULL if not available. 1806 * @addr: Virtual Address of the allocation. Must be inside the VMA. 1807 * 1808 * This function allocates a page from the kernel page pool and applies 1809 * a NUMA policy associated with the VMA or the current process. 1810 * When VMA is not NULL caller must hold down_read on the mmap_sem of the 1811 * mm_struct of the VMA to prevent it from going away. Should be used for 1812 * all allocations for pages that will be mapped into 1813 * user space. Returns NULL when no page can be allocated. 1814 * 1815 * Should be called with the mm_sem of the vma hold. 1816 */ 1817struct page * 1818alloc_page_vma(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr) 1819{ 1820 struct mempolicy *pol = get_vma_policy(current, vma, addr); 1821 struct zonelist *zl; 1822 struct page *page; 1823 1824 get_mems_allowed(); 1825 if (unlikely(pol->mode == MPOL_INTERLEAVE)) { 1826 unsigned nid; 1827 1828 nid = interleave_nid(pol, vma, addr, PAGE_SHIFT); 1829 mpol_cond_put(pol); 1830 page = alloc_page_interleave(gfp, 0, nid); 1831 put_mems_allowed(); 1832 return page; 1833 } 1834 zl = policy_zonelist(gfp, pol); 1835 if (unlikely(mpol_needs_cond_ref(pol))) { 1836 /* 1837 * slow path: ref counted shared policy 1838 */ 1839 struct page *page = __alloc_pages_nodemask(gfp, 0, 1840 zl, policy_nodemask(gfp, pol)); 1841 __mpol_put(pol); 1842 put_mems_allowed(); 1843 return page; 1844 } 1845 /* 1846 * fast path: default or task policy 1847 */ 1848 page = __alloc_pages_nodemask(gfp, 0, zl, policy_nodemask(gfp, pol)); 1849 put_mems_allowed(); 1850 return page; 1851} 1852 1853/** 1854 * alloc_pages_current - Allocate pages. 1855 * 1856 * @gfp: 1857 * %GFP_USER user allocation, 1858 * %GFP_KERNEL kernel allocation, 1859 * %GFP_HIGHMEM highmem allocation, 1860 * %GFP_FS don't call back into a file system. 1861 * %GFP_ATOMIC don't sleep. 1862 * @order: Power of two of allocation size in pages. 0 is a single page. 1863 * 1864 * Allocate a page from the kernel page pool. When not in 1865 * interrupt context and apply the current process NUMA policy. 1866 * Returns NULL when no page can be allocated. 1867 * 1868 * Don't call cpuset_update_task_memory_state() unless 1869 * 1) it's ok to take cpuset_sem (can WAIT), and 1870 * 2) allocating for current task (not interrupt). 1871 */ 1872struct page *alloc_pages_current(gfp_t gfp, unsigned order) 1873{ 1874 struct mempolicy *pol = current->mempolicy; 1875 struct page *page; 1876 1877 if (!pol || in_interrupt() || (gfp & __GFP_THISNODE)) 1878 pol = &default_policy; 1879 1880 get_mems_allowed(); 1881 /* 1882 * No reference counting needed for current->mempolicy 1883 * nor system default_policy 1884 */ 1885 if (pol->mode == MPOL_INTERLEAVE) 1886 page = alloc_page_interleave(gfp, order, interleave_nodes(pol)); 1887 else 1888 page = __alloc_pages_nodemask(gfp, order, 1889 policy_zonelist(gfp, pol), policy_nodemask(gfp, pol)); 1890 put_mems_allowed(); 1891 return page; 1892} 1893EXPORT_SYMBOL(alloc_pages_current); 1894 1895/* 1896 * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it 1897 * rebinds the mempolicy its copying by calling mpol_rebind_policy() 1898 * with the mems_allowed returned by cpuset_mems_allowed(). This 1899 * keeps mempolicies cpuset relative after its cpuset moves. See 1900 * further kernel/cpuset.c update_nodemask(). 1901 * 1902 * current's mempolicy may be rebinded by the other task(the task that changes 1903 * cpuset's mems), so we needn't do rebind work for current task. 1904 */ 1905 1906/* Slow path of a mempolicy duplicate */ 1907struct mempolicy *__mpol_dup(struct mempolicy *old) 1908{ 1909 struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL); 1910 1911 if (!new) 1912 return ERR_PTR(-ENOMEM); 1913 1914 /* task's mempolicy is protected by alloc_lock */ 1915 if (old == current->mempolicy) { 1916 task_lock(current); 1917 *new = *old; 1918 task_unlock(current); 1919 } else 1920 *new = *old; 1921 1922 rcu_read_lock(); 1923 if (current_cpuset_is_being_rebound()) { 1924 nodemask_t mems = cpuset_mems_allowed(current); 1925 if (new->flags & MPOL_F_REBINDING) 1926 mpol_rebind_policy(new, &mems, MPOL_REBIND_STEP2); 1927 else 1928 mpol_rebind_policy(new, &mems, MPOL_REBIND_ONCE); 1929 } 1930 rcu_read_unlock(); 1931 atomic_set(&new->refcnt, 1); 1932 return new; 1933} 1934 1935/* 1936 * If *frompol needs [has] an extra ref, copy *frompol to *tompol , 1937 * eliminate the * MPOL_F_* flags that require conditional ref and 1938 * [NOTE!!!] drop the extra ref. Not safe to reference *frompol directly 1939 * after return. Use the returned value. 1940 * 1941 * Allows use of a mempolicy for, e.g., multiple allocations with a single 1942 * policy lookup, even if the policy needs/has extra ref on lookup. 1943 * shmem_readahead needs this. 1944 */ 1945struct mempolicy *__mpol_cond_copy(struct mempolicy *tompol, 1946 struct mempolicy *frompol) 1947{ 1948 if (!mpol_needs_cond_ref(frompol)) 1949 return frompol; 1950 1951 *tompol = *frompol; 1952 tompol->flags &= ~MPOL_F_SHARED; /* copy doesn't need unref */ 1953 __mpol_put(frompol); 1954 return tompol; 1955} 1956 1957/* Slow path of a mempolicy comparison */ 1958int __mpol_equal(struct mempolicy *a, struct mempolicy *b) 1959{ 1960 if (!a || !b) 1961 return 0; 1962 if (a->mode != b->mode) 1963 return 0; 1964 if (a->flags != b->flags) 1965 return 0; 1966 if (mpol_store_user_nodemask(a)) 1967 if (!nodes_equal(a->w.user_nodemask, b->w.user_nodemask)) 1968 return 0; 1969 1970 switch (a->mode) { 1971 case MPOL_BIND: 1972 /* Fall through */ 1973 case MPOL_INTERLEAVE: 1974 return nodes_equal(a->v.nodes, b->v.nodes); 1975 case MPOL_PREFERRED: 1976 return a->v.preferred_node == b->v.preferred_node && 1977 a->flags == b->flags; 1978 default: 1979 BUG(); 1980 return 0; 1981 } 1982} 1983 1984/* 1985 * Shared memory backing store policy support. 1986 * 1987 * Remember policies even when nobody has shared memory mapped. 1988 * The policies are kept in Red-Black tree linked from the inode. 1989 * They are protected by the sp->lock spinlock, which should be held 1990 * for any accesses to the tree. 1991 */ 1992 1993/* lookup first element intersecting start-end */ 1994/* Caller holds sp->lock */ 1995static struct sp_node * 1996sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end) 1997{ 1998 struct rb_node *n = sp->root.rb_node; 1999 2000 while (n) { 2001 struct sp_node *p = rb_entry(n, struct sp_node, nd); 2002 2003 if (start >= p->end) 2004 n = n->rb_right; 2005 else if (end <= p->start) 2006 n = n->rb_left; 2007 else 2008 break; 2009 } 2010 if (!n) 2011 return NULL; 2012 for (;;) { 2013 struct sp_node *w = NULL; 2014 struct rb_node *prev = rb_prev(n); 2015 if (!prev) 2016 break; 2017 w = rb_entry(prev, struct sp_node, nd); 2018 if (w->end <= start) 2019 break; 2020 n = prev; 2021 } 2022 return rb_entry(n, struct sp_node, nd); 2023} 2024 2025/* Insert a new shared policy into the list. */ 2026/* Caller holds sp->lock */ 2027static void sp_insert(struct shared_policy *sp, struct sp_node *new) 2028{ 2029 struct rb_node **p = &sp->root.rb_node; 2030 struct rb_node *parent = NULL; 2031 struct sp_node *nd; 2032 2033 while (*p) { 2034 parent = *p; 2035 nd = rb_entry(parent, struct sp_node, nd); 2036 if (new->start < nd->start) 2037 p = &(*p)->rb_left; 2038 else if (new->end > nd->end) 2039 p = &(*p)->rb_right; 2040 else 2041 BUG(); 2042 } 2043 rb_link_node(&new->nd, parent, p); 2044 rb_insert_color(&new->nd, &sp->root); 2045 pr_debug("inserting %lx-%lx: %d\n", new->start, new->end, 2046 new->policy ? new->policy->mode : 0); 2047} 2048 2049/* Find shared policy intersecting idx */ 2050struct mempolicy * 2051mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx) 2052{ 2053 struct mempolicy *pol = NULL; 2054 struct sp_node *sn; 2055 2056 if (!sp->root.rb_node) 2057 return NULL; 2058 spin_lock(&sp->lock); 2059 sn = sp_lookup(sp, idx, idx+1); 2060 if (sn) { 2061 mpol_get(sn->policy); 2062 pol = sn->policy; 2063 } 2064 spin_unlock(&sp->lock); 2065 return pol; 2066} 2067 2068static void sp_delete(struct shared_policy *sp, struct sp_node *n) 2069{ 2070 pr_debug("deleting %lx-l%lx\n", n->start, n->end); 2071 rb_erase(&n->nd, &sp->root); 2072 mpol_put(n->policy); 2073 kmem_cache_free(sn_cache, n); 2074} 2075 2076static struct sp_node *sp_alloc(unsigned long start, unsigned long end, 2077 struct mempolicy *pol) 2078{ 2079 struct sp_node *n = kmem_cache_alloc(sn_cache, GFP_KERNEL); 2080 2081 if (!n) 2082 return NULL; 2083 n->start = start; 2084 n->end = end; 2085 mpol_get(pol); 2086 pol->flags |= MPOL_F_SHARED; /* for unref */ 2087 n->policy = pol; 2088 return n; 2089} 2090 2091/* Replace a policy range. */ 2092static int shared_policy_replace(struct shared_policy *sp, unsigned long start, 2093 unsigned long end, struct sp_node *new) 2094{ 2095 struct sp_node *n, *new2 = NULL; 2096 2097restart: 2098 spin_lock(&sp->lock); 2099 n = sp_lookup(sp, start, end); 2100 /* Take care of old policies in the same range. */ 2101 while (n && n->start < end) { 2102 struct rb_node *next = rb_next(&n->nd); 2103 if (n->start >= start) { 2104 if (n->end <= end) 2105 sp_delete(sp, n); 2106 else 2107 n->start = end; 2108 } else { 2109 /* Old policy spanning whole new range. */ 2110 if (n->end > end) { 2111 if (!new2) { 2112 spin_unlock(&sp->lock); 2113 new2 = sp_alloc(end, n->end, n->policy); 2114 if (!new2) 2115 return -ENOMEM; 2116 goto restart; 2117 } 2118 n->end = start; 2119 sp_insert(sp, new2); 2120 new2 = NULL; 2121 break; 2122 } else 2123 n->end = start; 2124 } 2125 if (!next) 2126 break; 2127 n = rb_entry(next, struct sp_node, nd); 2128 } 2129 if (new) 2130 sp_insert(sp, new); 2131 spin_unlock(&sp->lock); 2132 if (new2) { 2133 mpol_put(new2->policy); 2134 kmem_cache_free(sn_cache, new2); 2135 } 2136 return 0; 2137} 2138 2139/** 2140 * mpol_shared_policy_init - initialize shared policy for inode 2141 * @sp: pointer to inode shared policy 2142 * @mpol: struct mempolicy to install 2143 * 2144 * Install non-NULL @mpol in inode's shared policy rb-tree. 2145 * On entry, the current task has a reference on a non-NULL @mpol. 2146 * This must be released on exit. 2147 * This is called at get_inode() calls and we can use GFP_KERNEL. 2148 */ 2149void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol) 2150{ 2151 int ret; 2152 2153 sp->root = RB_ROOT; /* empty tree == default mempolicy */ 2154 spin_lock_init(&sp->lock); 2155 2156 if (mpol) { 2157 struct vm_area_struct pvma; 2158 struct mempolicy *new; 2159 NODEMASK_SCRATCH(scratch); 2160 2161 if (!scratch) 2162 goto put_mpol; 2163 /* contextualize the tmpfs mount point mempolicy */ 2164 new = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask); 2165 if (IS_ERR(new)) 2166 goto free_scratch; /* no valid nodemask intersection */ 2167 2168 task_lock(current); 2169 ret = mpol_set_nodemask(new, &mpol->w.user_nodemask, scratch); 2170 task_unlock(current); 2171 if (ret) 2172 goto put_new; 2173 2174 /* Create pseudo-vma that contains just the policy */ 2175 memset(&pvma, 0, sizeof(struct vm_area_struct)); 2176 pvma.vm_end = TASK_SIZE; /* policy covers entire file */ 2177 mpol_set_shared_policy(sp, &pvma, new); /* adds ref */ 2178 2179put_new: 2180 mpol_put(new); /* drop initial ref */ 2181free_scratch: 2182 NODEMASK_SCRATCH_FREE(scratch); 2183put_mpol: 2184 mpol_put(mpol); /* drop our incoming ref on sb mpol */ 2185 } 2186} 2187 2188int mpol_set_shared_policy(struct shared_policy *info, 2189 struct vm_area_struct *vma, struct mempolicy *npol) 2190{ 2191 int err; 2192 struct sp_node *new = NULL; 2193 unsigned long sz = vma_pages(vma); 2194 2195 pr_debug("set_shared_policy %lx sz %lu %d %d %lx\n", 2196 vma->vm_pgoff, 2197 sz, npol ? npol->mode : -1, 2198 npol ? npol->flags : -1, 2199 npol ? nodes_addr(npol->v.nodes)[0] : -1); 2200 2201 if (npol) { 2202 new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol); 2203 if (!new) 2204 return -ENOMEM; 2205 } 2206 err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new); 2207 if (err && new) 2208 kmem_cache_free(sn_cache, new); 2209 return err; 2210} 2211 2212/* Free a backing policy store on inode delete. */ 2213void mpol_free_shared_policy(struct shared_policy *p) 2214{ 2215 struct sp_node *n; 2216 struct rb_node *next; 2217 2218 if (!p->root.rb_node) 2219 return; 2220 spin_lock(&p->lock); 2221 next = rb_first(&p->root); 2222 while (next) { 2223 n = rb_entry(next, struct sp_node, nd); 2224 next = rb_next(&n->nd); 2225 rb_erase(&n->nd, &p->root); 2226 mpol_put(n->policy); 2227 kmem_cache_free(sn_cache, n); 2228 } 2229 spin_unlock(&p->lock); 2230} 2231 2232/* assumes fs == KERNEL_DS */ 2233void __init numa_policy_init(void) 2234{ 2235 nodemask_t interleave_nodes; 2236 unsigned long largest = 0; 2237 int nid, prefer = 0; 2238 2239 policy_cache = kmem_cache_create("numa_policy", 2240 sizeof(struct mempolicy), 2241 0, SLAB_PANIC, NULL); 2242 2243 sn_cache = kmem_cache_create("shared_policy_node", 2244 sizeof(struct sp_node), 2245 0, SLAB_PANIC, NULL); 2246 2247 /* 2248 * Set interleaving policy for system init. Interleaving is only 2249 * enabled across suitably sized nodes (default is >= 16MB), or 2250 * fall back to the largest node if they're all smaller. 2251 */ 2252 nodes_clear(interleave_nodes); 2253 for_each_node_state(nid, N_HIGH_MEMORY) { 2254 unsigned long total_pages = node_present_pages(nid); 2255 2256 /* Preserve the largest node */ 2257 if (largest < total_pages) { 2258 largest = total_pages; 2259 prefer = nid; 2260 } 2261 2262 /* Interleave this node? */ 2263 if ((total_pages << PAGE_SHIFT) >= (16 << 20)) 2264 node_set(nid, interleave_nodes); 2265 } 2266 2267 /* All too small, use the largest */ 2268 if (unlikely(nodes_empty(interleave_nodes))) 2269 node_set(prefer, interleave_nodes); 2270 2271 if (do_set_mempolicy(MPOL_INTERLEAVE, 0, &interleave_nodes)) 2272 printk("numa_policy_init: interleaving failed\n"); 2273} 2274 2275/* Reset policy of current process to default */ 2276void numa_default_policy(void) 2277{ 2278 do_set_mempolicy(MPOL_DEFAULT, 0, NULL); 2279} 2280 2281/* 2282 * Parse and format mempolicy from/to strings 2283 */ 2284 2285/* 2286 * "local" is pseudo-policy: MPOL_PREFERRED with MPOL_F_LOCAL flag 2287 * Used only for mpol_parse_str() and mpol_to_str() 2288 */ 2289#define MPOL_LOCAL MPOL_MAX 2290static const char * const policy_modes[] = 2291{ 2292 [MPOL_DEFAULT] = "default", 2293 [MPOL_PREFERRED] = "prefer", 2294 [MPOL_BIND] = "bind", 2295 [MPOL_INTERLEAVE] = "interleave", 2296 [MPOL_LOCAL] = "local" 2297}; 2298 2299 2300#ifdef CONFIG_TMPFS 2301/** 2302 * mpol_parse_str - parse string to mempolicy 2303 * @str: string containing mempolicy to parse 2304 * @mpol: pointer to struct mempolicy pointer, returned on success. 2305 * @no_context: flag whether to "contextualize" the mempolicy 2306 * 2307 * Format of input: 2308 * <mode>[=<flags>][:<nodelist>] 2309 * 2310 * if @no_context is true, save the input nodemask in w.user_nodemask in 2311 * the returned mempolicy. This will be used to "clone" the mempolicy in 2312 * a specific context [cpuset] at a later time. Used to parse tmpfs mpol 2313 * mount option. Note that if 'static' or 'relative' mode flags were 2314 * specified, the input nodemask will already have been saved. Saving 2315 * it again is redundant, but safe. 2316 * 2317 * On success, returns 0, else 1 2318 */ 2319int mpol_parse_str(char *str, struct mempolicy **mpol, int no_context) 2320{ 2321 struct mempolicy *new = NULL; 2322 unsigned short mode; 2323 unsigned short uninitialized_var(mode_flags); 2324 nodemask_t nodes; 2325 char *nodelist = strchr(str, ':'); 2326 char *flags = strchr(str, '='); 2327 int err = 1; 2328 2329 if (nodelist) { 2330 /* NUL-terminate mode or flags string */ 2331 *nodelist++ = '\0'; 2332 if (nodelist_parse(nodelist, nodes)) 2333 goto out; 2334 if (!nodes_subset(nodes, node_states[N_HIGH_MEMORY])) 2335 goto out; 2336 } else 2337 nodes_clear(nodes); 2338 2339 if (flags) 2340 *flags++ = '\0'; /* terminate mode string */ 2341 2342 for (mode = 0; mode <= MPOL_LOCAL; mode++) { 2343 if (!strcmp(str, policy_modes[mode])) { 2344 break; 2345 } 2346 } 2347 if (mode > MPOL_LOCAL) 2348 goto out; 2349 2350 switch (mode) { 2351 case MPOL_PREFERRED: 2352 /* 2353 * Insist on a nodelist of one node only 2354 */ 2355 if (nodelist) { 2356 char *rest = nodelist; 2357 while (isdigit(*rest)) 2358 rest++; 2359 if (*rest) 2360 goto out; 2361 } 2362 break; 2363 case MPOL_INTERLEAVE: 2364 /* 2365 * Default to online nodes with memory if no nodelist 2366 */ 2367 if (!nodelist) 2368 nodes = node_states[N_HIGH_MEMORY]; 2369 break; 2370 case MPOL_LOCAL: 2371 /* 2372 * Don't allow a nodelist; mpol_new() checks flags 2373 */ 2374 if (nodelist) 2375 goto out; 2376 mode = MPOL_PREFERRED; 2377 break; 2378 case MPOL_DEFAULT: 2379 /* 2380 * Insist on a empty nodelist 2381 */ 2382 if (!nodelist) 2383 err = 0; 2384 goto out; 2385 case MPOL_BIND: 2386 /* 2387 * Insist on a nodelist 2388 */ 2389 if (!nodelist) 2390 goto out; 2391 } 2392 2393 mode_flags = 0; 2394 if (flags) { 2395 /* 2396 * Currently, we only support two mutually exclusive 2397 * mode flags. 2398 */ 2399 if (!strcmp(flags, "static")) 2400 mode_flags |= MPOL_F_STATIC_NODES; 2401 else if (!strcmp(flags, "relative")) 2402 mode_flags |= MPOL_F_RELATIVE_NODES; 2403 else 2404 goto out; 2405 } 2406 2407 new = mpol_new(mode, mode_flags, &nodes); 2408 if (IS_ERR(new)) 2409 goto out; 2410 2411 if (no_context) { 2412 /* save for contextualization */ 2413 new->w.user_nodemask = nodes; 2414 } else { 2415 int ret; 2416 NODEMASK_SCRATCH(scratch); 2417 if (scratch) { 2418 task_lock(current); 2419 ret = mpol_set_nodemask(new, &nodes, scratch); 2420 task_unlock(current); 2421 } else 2422 ret = -ENOMEM; 2423 NODEMASK_SCRATCH_FREE(scratch); 2424 if (ret) { 2425 mpol_put(new); 2426 goto out; 2427 } 2428 } 2429 err = 0; 2430 2431out: 2432 /* Restore string for error message */ 2433 if (nodelist) 2434 *--nodelist = ':'; 2435 if (flags) 2436 *--flags = '='; 2437 if (!err) 2438 *mpol = new; 2439 return err; 2440} 2441#endif /* CONFIG_TMPFS */ 2442 2443/** 2444 * mpol_to_str - format a mempolicy structure for printing 2445 * @buffer: to contain formatted mempolicy string 2446 * @maxlen: length of @buffer 2447 * @pol: pointer to mempolicy to be formatted 2448 * @no_context: "context free" mempolicy - use nodemask in w.user_nodemask 2449 * 2450 * Convert a mempolicy into a string. 2451 * Returns the number of characters in buffer (if positive) 2452 * or an error (negative) 2453 */ 2454int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol, int no_context) 2455{ 2456 char *p = buffer; 2457 int l; 2458 nodemask_t nodes; 2459 unsigned short mode; 2460 unsigned short flags = pol ? pol->flags : 0; 2461 2462 /* 2463 * Sanity check: room for longest mode, flag and some nodes 2464 */ 2465 VM_BUG_ON(maxlen < strlen("interleave") + strlen("relative") + 16); 2466 2467 if (!pol || pol == &default_policy) 2468 mode = MPOL_DEFAULT; 2469 else 2470 mode = pol->mode; 2471 2472 switch (mode) { 2473 case MPOL_DEFAULT: 2474 nodes_clear(nodes); 2475 break; 2476 2477 case MPOL_PREFERRED: 2478 nodes_clear(nodes); 2479 if (flags & MPOL_F_LOCAL) 2480 mode = MPOL_LOCAL; /* pseudo-policy */ 2481 else 2482 node_set(pol->v.preferred_node, nodes); 2483 break; 2484 2485 case MPOL_BIND: 2486 /* Fall through */ 2487 case MPOL_INTERLEAVE: 2488 if (no_context) 2489 nodes = pol->w.user_nodemask; 2490 else 2491 nodes = pol->v.nodes; 2492 break; 2493 2494 default: 2495 BUG(); 2496 } 2497 2498 l = strlen(policy_modes[mode]); 2499 if (buffer + maxlen < p + l + 1) 2500 return -ENOSPC; 2501 2502 strcpy(p, policy_modes[mode]); 2503 p += l; 2504 2505 if (flags & MPOL_MODE_FLAGS) { 2506 if (buffer + maxlen < p + 2) 2507 return -ENOSPC; 2508 *p++ = '='; 2509 2510 /* 2511 * Currently, the only defined flags are mutually exclusive 2512 */ 2513 if (flags & MPOL_F_STATIC_NODES) 2514 p += snprintf(p, buffer + maxlen - p, "static"); 2515 else if (flags & MPOL_F_RELATIVE_NODES) 2516 p += snprintf(p, buffer + maxlen - p, "relative"); 2517 } 2518 2519 if (!nodes_empty(nodes)) { 2520 if (buffer + maxlen < p + 2) 2521 return -ENOSPC; 2522 *p++ = ':'; 2523 p += nodelist_scnprintf(p, buffer + maxlen - p, nodes); 2524 } 2525 return p - buffer; 2526} 2527 2528struct numa_maps { 2529 unsigned long pages; 2530 unsigned long anon; 2531 unsigned long active; 2532 unsigned long writeback; 2533 unsigned long mapcount_max; 2534 unsigned long dirty; 2535 unsigned long swapcache; 2536 unsigned long node[MAX_NUMNODES]; 2537}; 2538 2539static void gather_stats(struct page *page, void *private, int pte_dirty) 2540{ 2541 struct numa_maps *md = private; 2542 int count = page_mapcount(page); 2543 2544 md->pages++; 2545 if (pte_dirty || PageDirty(page)) 2546 md->dirty++; 2547 2548 if (PageSwapCache(page)) 2549 md->swapcache++; 2550 2551 if (PageActive(page) || PageUnevictable(page)) 2552 md->active++; 2553 2554 if (PageWriteback(page)) 2555 md->writeback++; 2556 2557 if (PageAnon(page)) 2558 md->anon++; 2559 2560 if (count > md->mapcount_max) 2561 md->mapcount_max = count; 2562 2563 md->node[page_to_nid(page)]++; 2564} 2565 2566#ifdef CONFIG_HUGETLB_PAGE 2567static void check_huge_range(struct vm_area_struct *vma, 2568 unsigned long start, unsigned long end, 2569 struct numa_maps *md) 2570{ 2571 unsigned long addr; 2572 struct page *page; 2573 struct hstate *h = hstate_vma(vma); 2574 unsigned long sz = huge_page_size(h); 2575 2576 for (addr = start; addr < end; addr += sz) { 2577 pte_t *ptep = huge_pte_offset(vma->vm_mm, 2578 addr & huge_page_mask(h)); 2579 pte_t pte; 2580 2581 if (!ptep) 2582 continue; 2583 2584 pte = *ptep; 2585 if (pte_none(pte)) 2586 continue; 2587 2588 page = pte_page(pte); 2589 if (!page) 2590 continue; 2591 2592 gather_stats(page, md, pte_dirty(*ptep)); 2593 } 2594} 2595#else 2596static inline void check_huge_range(struct vm_area_struct *vma, 2597 unsigned long start, unsigned long end, 2598 struct numa_maps *md) 2599{ 2600} 2601#endif 2602 2603/* 2604 * Display pages allocated per node and memory policy via /proc. 2605 */ 2606int show_numa_map(struct seq_file *m, void *v) 2607{ 2608 struct proc_maps_private *priv = m->private; 2609 struct vm_area_struct *vma = v; 2610 struct numa_maps *md; 2611 struct file *file = vma->vm_file; 2612 struct mm_struct *mm = vma->vm_mm; 2613 struct mempolicy *pol; 2614 int n; 2615 char buffer[50]; 2616 2617 if (!mm) 2618 return 0; 2619 2620 md = kzalloc(sizeof(struct numa_maps), GFP_KERNEL); 2621 if (!md) 2622 return 0; 2623 2624 pol = get_vma_policy(priv->task, vma, vma->vm_start); 2625 mpol_to_str(buffer, sizeof(buffer), pol, 0); 2626 mpol_cond_put(pol); 2627 2628 seq_printf(m, "%08lx %s", vma->vm_start, buffer); 2629 2630 if (file) { 2631 seq_printf(m, " file="); 2632 seq_path(m, &file->f_path, "\n\t= "); 2633 } else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) { 2634 seq_printf(m, " heap"); 2635 } else if (vma->vm_start <= mm->start_stack && 2636 vma->vm_end >= mm->start_stack) { 2637 seq_printf(m, " stack"); 2638 } 2639 2640 if (is_vm_hugetlb_page(vma)) { 2641 check_huge_range(vma, vma->vm_start, vma->vm_end, md); 2642 seq_printf(m, " huge"); 2643 } else { 2644 check_pgd_range(vma, vma->vm_start, vma->vm_end, 2645 &node_states[N_HIGH_MEMORY], MPOL_MF_STATS, md); 2646 } 2647 2648 if (!md->pages) 2649 goto out; 2650 2651 if (md->anon) 2652 seq_printf(m," anon=%lu",md->anon); 2653 2654 if (md->dirty) 2655 seq_printf(m," dirty=%lu",md->dirty); 2656 2657 if (md->pages != md->anon && md->pages != md->dirty) 2658 seq_printf(m, " mapped=%lu", md->pages); 2659 2660 if (md->mapcount_max > 1) 2661 seq_printf(m, " mapmax=%lu", md->mapcount_max); 2662 2663 if (md->swapcache) 2664 seq_printf(m," swapcache=%lu", md->swapcache); 2665 2666 if (md->active < md->pages && !is_vm_hugetlb_page(vma)) 2667 seq_printf(m," active=%lu", md->active); 2668 2669 if (md->writeback) 2670 seq_printf(m," writeback=%lu", md->writeback); 2671 2672 for_each_node_state(n, N_HIGH_MEMORY) 2673 if (md->node[n]) 2674 seq_printf(m, " N%d=%lu", n, md->node[n]); 2675out: 2676 seq_putc(m, '\n'); 2677 kfree(md); 2678 2679 if (m->count < m->size) 2680 m->version = (vma != priv->tail_vma) ? vma->vm_start : 0; 2681 return 0; 2682} 2683