memcontrol.c revision 5564e88ba6fd2f6dcd83a592771810cd84b5ae80
1/* memcontrol.c - Memory Controller 2 * 3 * Copyright IBM Corporation, 2007 4 * Author Balbir Singh <balbir@linux.vnet.ibm.com> 5 * 6 * Copyright 2007 OpenVZ SWsoft Inc 7 * Author: Pavel Emelianov <xemul@openvz.org> 8 * 9 * Memory thresholds 10 * Copyright (C) 2009 Nokia Corporation 11 * Author: Kirill A. Shutemov 12 * 13 * This program is free software; you can redistribute it and/or modify 14 * it under the terms of the GNU General Public License as published by 15 * the Free Software Foundation; either version 2 of the License, or 16 * (at your option) any later version. 17 * 18 * This program is distributed in the hope that it will be useful, 19 * but WITHOUT ANY WARRANTY; without even the implied warranty of 20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 21 * GNU General Public License for more details. 22 */ 23 24#include <linux/res_counter.h> 25#include <linux/memcontrol.h> 26#include <linux/cgroup.h> 27#include <linux/mm.h> 28#include <linux/hugetlb.h> 29#include <linux/pagemap.h> 30#include <linux/smp.h> 31#include <linux/page-flags.h> 32#include <linux/backing-dev.h> 33#include <linux/bit_spinlock.h> 34#include <linux/rcupdate.h> 35#include <linux/limits.h> 36#include <linux/mutex.h> 37#include <linux/rbtree.h> 38#include <linux/slab.h> 39#include <linux/swap.h> 40#include <linux/swapops.h> 41#include <linux/spinlock.h> 42#include <linux/eventfd.h> 43#include <linux/sort.h> 44#include <linux/fs.h> 45#include <linux/seq_file.h> 46#include <linux/vmalloc.h> 47#include <linux/mm_inline.h> 48#include <linux/page_cgroup.h> 49#include <linux/cpu.h> 50#include <linux/oom.h> 51#include "internal.h" 52 53#include <asm/uaccess.h> 54 55#include <trace/events/vmscan.h> 56 57struct cgroup_subsys mem_cgroup_subsys __read_mostly; 58#define MEM_CGROUP_RECLAIM_RETRIES 5 59struct mem_cgroup *root_mem_cgroup __read_mostly; 60 61#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP 62/* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */ 63int do_swap_account __read_mostly; 64 65/* for remember boot option*/ 66#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP_ENABLED 67static int really_do_swap_account __initdata = 1; 68#else 69static int really_do_swap_account __initdata = 0; 70#endif 71 72#else 73#define do_swap_account (0) 74#endif 75 76/* 77 * Per memcg event counter is incremented at every pagein/pageout. This counter 78 * is used for trigger some periodic events. This is straightforward and better 79 * than using jiffies etc. to handle periodic memcg event. 80 * 81 * These values will be used as !((event) & ((1 <<(thresh)) - 1)) 82 */ 83#define THRESHOLDS_EVENTS_THRESH (7) /* once in 128 */ 84#define SOFTLIMIT_EVENTS_THRESH (10) /* once in 1024 */ 85 86/* 87 * Statistics for memory cgroup. 88 */ 89enum mem_cgroup_stat_index { 90 /* 91 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss. 92 */ 93 MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */ 94 MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */ 95 MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */ 96 MEM_CGROUP_STAT_PGPGIN_COUNT, /* # of pages paged in */ 97 MEM_CGROUP_STAT_PGPGOUT_COUNT, /* # of pages paged out */ 98 MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */ 99 MEM_CGROUP_STAT_DATA, /* end of data requires synchronization */ 100 /* incremented at every pagein/pageout */ 101 MEM_CGROUP_EVENTS = MEM_CGROUP_STAT_DATA, 102 MEM_CGROUP_ON_MOVE, /* someone is moving account between groups */ 103 104 MEM_CGROUP_STAT_NSTATS, 105}; 106 107struct mem_cgroup_stat_cpu { 108 s64 count[MEM_CGROUP_STAT_NSTATS]; 109}; 110 111/* 112 * per-zone information in memory controller. 113 */ 114struct mem_cgroup_per_zone { 115 /* 116 * spin_lock to protect the per cgroup LRU 117 */ 118 struct list_head lists[NR_LRU_LISTS]; 119 unsigned long count[NR_LRU_LISTS]; 120 121 struct zone_reclaim_stat reclaim_stat; 122 struct rb_node tree_node; /* RB tree node */ 123 unsigned long long usage_in_excess;/* Set to the value by which */ 124 /* the soft limit is exceeded*/ 125 bool on_tree; 126 struct mem_cgroup *mem; /* Back pointer, we cannot */ 127 /* use container_of */ 128}; 129/* Macro for accessing counter */ 130#define MEM_CGROUP_ZSTAT(mz, idx) ((mz)->count[(idx)]) 131 132struct mem_cgroup_per_node { 133 struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES]; 134}; 135 136struct mem_cgroup_lru_info { 137 struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES]; 138}; 139 140/* 141 * Cgroups above their limits are maintained in a RB-Tree, independent of 142 * their hierarchy representation 143 */ 144 145struct mem_cgroup_tree_per_zone { 146 struct rb_root rb_root; 147 spinlock_t lock; 148}; 149 150struct mem_cgroup_tree_per_node { 151 struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES]; 152}; 153 154struct mem_cgroup_tree { 155 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES]; 156}; 157 158static struct mem_cgroup_tree soft_limit_tree __read_mostly; 159 160struct mem_cgroup_threshold { 161 struct eventfd_ctx *eventfd; 162 u64 threshold; 163}; 164 165/* For threshold */ 166struct mem_cgroup_threshold_ary { 167 /* An array index points to threshold just below usage. */ 168 int current_threshold; 169 /* Size of entries[] */ 170 unsigned int size; 171 /* Array of thresholds */ 172 struct mem_cgroup_threshold entries[0]; 173}; 174 175struct mem_cgroup_thresholds { 176 /* Primary thresholds array */ 177 struct mem_cgroup_threshold_ary *primary; 178 /* 179 * Spare threshold array. 180 * This is needed to make mem_cgroup_unregister_event() "never fail". 181 * It must be able to store at least primary->size - 1 entries. 182 */ 183 struct mem_cgroup_threshold_ary *spare; 184}; 185 186/* for OOM */ 187struct mem_cgroup_eventfd_list { 188 struct list_head list; 189 struct eventfd_ctx *eventfd; 190}; 191 192static void mem_cgroup_threshold(struct mem_cgroup *mem); 193static void mem_cgroup_oom_notify(struct mem_cgroup *mem); 194 195/* 196 * The memory controller data structure. The memory controller controls both 197 * page cache and RSS per cgroup. We would eventually like to provide 198 * statistics based on the statistics developed by Rik Van Riel for clock-pro, 199 * to help the administrator determine what knobs to tune. 200 * 201 * TODO: Add a water mark for the memory controller. Reclaim will begin when 202 * we hit the water mark. May be even add a low water mark, such that 203 * no reclaim occurs from a cgroup at it's low water mark, this is 204 * a feature that will be implemented much later in the future. 205 */ 206struct mem_cgroup { 207 struct cgroup_subsys_state css; 208 /* 209 * the counter to account for memory usage 210 */ 211 struct res_counter res; 212 /* 213 * the counter to account for mem+swap usage. 214 */ 215 struct res_counter memsw; 216 /* 217 * Per cgroup active and inactive list, similar to the 218 * per zone LRU lists. 219 */ 220 struct mem_cgroup_lru_info info; 221 222 /* 223 protect against reclaim related member. 224 */ 225 spinlock_t reclaim_param_lock; 226 227 /* 228 * While reclaiming in a hierarchy, we cache the last child we 229 * reclaimed from. 230 */ 231 int last_scanned_child; 232 /* 233 * Should the accounting and control be hierarchical, per subtree? 234 */ 235 bool use_hierarchy; 236 atomic_t oom_lock; 237 atomic_t refcnt; 238 239 unsigned int swappiness; 240 /* OOM-Killer disable */ 241 int oom_kill_disable; 242 243 /* set when res.limit == memsw.limit */ 244 bool memsw_is_minimum; 245 246 /* protect arrays of thresholds */ 247 struct mutex thresholds_lock; 248 249 /* thresholds for memory usage. RCU-protected */ 250 struct mem_cgroup_thresholds thresholds; 251 252 /* thresholds for mem+swap usage. RCU-protected */ 253 struct mem_cgroup_thresholds memsw_thresholds; 254 255 /* For oom notifier event fd */ 256 struct list_head oom_notify; 257 258 /* 259 * Should we move charges of a task when a task is moved into this 260 * mem_cgroup ? And what type of charges should we move ? 261 */ 262 unsigned long move_charge_at_immigrate; 263 /* 264 * percpu counter. 265 */ 266 struct mem_cgroup_stat_cpu *stat; 267 /* 268 * used when a cpu is offlined or other synchronizations 269 * See mem_cgroup_read_stat(). 270 */ 271 struct mem_cgroup_stat_cpu nocpu_base; 272 spinlock_t pcp_counter_lock; 273}; 274 275/* Stuffs for move charges at task migration. */ 276/* 277 * Types of charges to be moved. "move_charge_at_immitgrate" is treated as a 278 * left-shifted bitmap of these types. 279 */ 280enum move_type { 281 MOVE_CHARGE_TYPE_ANON, /* private anonymous page and swap of it */ 282 MOVE_CHARGE_TYPE_FILE, /* file page(including tmpfs) and swap of it */ 283 NR_MOVE_TYPE, 284}; 285 286/* "mc" and its members are protected by cgroup_mutex */ 287static struct move_charge_struct { 288 spinlock_t lock; /* for from, to */ 289 struct mem_cgroup *from; 290 struct mem_cgroup *to; 291 unsigned long precharge; 292 unsigned long moved_charge; 293 unsigned long moved_swap; 294 struct task_struct *moving_task; /* a task moving charges */ 295 wait_queue_head_t waitq; /* a waitq for other context */ 296} mc = { 297 .lock = __SPIN_LOCK_UNLOCKED(mc.lock), 298 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq), 299}; 300 301static bool move_anon(void) 302{ 303 return test_bit(MOVE_CHARGE_TYPE_ANON, 304 &mc.to->move_charge_at_immigrate); 305} 306 307static bool move_file(void) 308{ 309 return test_bit(MOVE_CHARGE_TYPE_FILE, 310 &mc.to->move_charge_at_immigrate); 311} 312 313/* 314 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft 315 * limit reclaim to prevent infinite loops, if they ever occur. 316 */ 317#define MEM_CGROUP_MAX_RECLAIM_LOOPS (100) 318#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS (2) 319 320enum charge_type { 321 MEM_CGROUP_CHARGE_TYPE_CACHE = 0, 322 MEM_CGROUP_CHARGE_TYPE_MAPPED, 323 MEM_CGROUP_CHARGE_TYPE_SHMEM, /* used by page migration of shmem */ 324 MEM_CGROUP_CHARGE_TYPE_FORCE, /* used by force_empty */ 325 MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */ 326 MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */ 327 NR_CHARGE_TYPE, 328}; 329 330/* for encoding cft->private value on file */ 331#define _MEM (0) 332#define _MEMSWAP (1) 333#define _OOM_TYPE (2) 334#define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val)) 335#define MEMFILE_TYPE(val) (((val) >> 16) & 0xffff) 336#define MEMFILE_ATTR(val) ((val) & 0xffff) 337/* Used for OOM nofiier */ 338#define OOM_CONTROL (0) 339 340/* 341 * Reclaim flags for mem_cgroup_hierarchical_reclaim 342 */ 343#define MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0 344#define MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT) 345#define MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1 346#define MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT) 347#define MEM_CGROUP_RECLAIM_SOFT_BIT 0x2 348#define MEM_CGROUP_RECLAIM_SOFT (1 << MEM_CGROUP_RECLAIM_SOFT_BIT) 349 350static void mem_cgroup_get(struct mem_cgroup *mem); 351static void mem_cgroup_put(struct mem_cgroup *mem); 352static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem); 353static void drain_all_stock_async(void); 354 355static struct mem_cgroup_per_zone * 356mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid) 357{ 358 return &mem->info.nodeinfo[nid]->zoneinfo[zid]; 359} 360 361struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem) 362{ 363 return &mem->css; 364} 365 366static struct mem_cgroup_per_zone * 367page_cgroup_zoneinfo(struct mem_cgroup *mem, struct page *page) 368{ 369 int nid = page_to_nid(page); 370 int zid = page_zonenum(page); 371 372 return mem_cgroup_zoneinfo(mem, nid, zid); 373} 374 375static struct mem_cgroup_tree_per_zone * 376soft_limit_tree_node_zone(int nid, int zid) 377{ 378 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid]; 379} 380 381static struct mem_cgroup_tree_per_zone * 382soft_limit_tree_from_page(struct page *page) 383{ 384 int nid = page_to_nid(page); 385 int zid = page_zonenum(page); 386 387 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid]; 388} 389 390static void 391__mem_cgroup_insert_exceeded(struct mem_cgroup *mem, 392 struct mem_cgroup_per_zone *mz, 393 struct mem_cgroup_tree_per_zone *mctz, 394 unsigned long long new_usage_in_excess) 395{ 396 struct rb_node **p = &mctz->rb_root.rb_node; 397 struct rb_node *parent = NULL; 398 struct mem_cgroup_per_zone *mz_node; 399 400 if (mz->on_tree) 401 return; 402 403 mz->usage_in_excess = new_usage_in_excess; 404 if (!mz->usage_in_excess) 405 return; 406 while (*p) { 407 parent = *p; 408 mz_node = rb_entry(parent, struct mem_cgroup_per_zone, 409 tree_node); 410 if (mz->usage_in_excess < mz_node->usage_in_excess) 411 p = &(*p)->rb_left; 412 /* 413 * We can't avoid mem cgroups that are over their soft 414 * limit by the same amount 415 */ 416 else if (mz->usage_in_excess >= mz_node->usage_in_excess) 417 p = &(*p)->rb_right; 418 } 419 rb_link_node(&mz->tree_node, parent, p); 420 rb_insert_color(&mz->tree_node, &mctz->rb_root); 421 mz->on_tree = true; 422} 423 424static void 425__mem_cgroup_remove_exceeded(struct mem_cgroup *mem, 426 struct mem_cgroup_per_zone *mz, 427 struct mem_cgroup_tree_per_zone *mctz) 428{ 429 if (!mz->on_tree) 430 return; 431 rb_erase(&mz->tree_node, &mctz->rb_root); 432 mz->on_tree = false; 433} 434 435static void 436mem_cgroup_remove_exceeded(struct mem_cgroup *mem, 437 struct mem_cgroup_per_zone *mz, 438 struct mem_cgroup_tree_per_zone *mctz) 439{ 440 spin_lock(&mctz->lock); 441 __mem_cgroup_remove_exceeded(mem, mz, mctz); 442 spin_unlock(&mctz->lock); 443} 444 445 446static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page) 447{ 448 unsigned long long excess; 449 struct mem_cgroup_per_zone *mz; 450 struct mem_cgroup_tree_per_zone *mctz; 451 int nid = page_to_nid(page); 452 int zid = page_zonenum(page); 453 mctz = soft_limit_tree_from_page(page); 454 455 /* 456 * Necessary to update all ancestors when hierarchy is used. 457 * because their event counter is not touched. 458 */ 459 for (; mem; mem = parent_mem_cgroup(mem)) { 460 mz = mem_cgroup_zoneinfo(mem, nid, zid); 461 excess = res_counter_soft_limit_excess(&mem->res); 462 /* 463 * We have to update the tree if mz is on RB-tree or 464 * mem is over its softlimit. 465 */ 466 if (excess || mz->on_tree) { 467 spin_lock(&mctz->lock); 468 /* if on-tree, remove it */ 469 if (mz->on_tree) 470 __mem_cgroup_remove_exceeded(mem, mz, mctz); 471 /* 472 * Insert again. mz->usage_in_excess will be updated. 473 * If excess is 0, no tree ops. 474 */ 475 __mem_cgroup_insert_exceeded(mem, mz, mctz, excess); 476 spin_unlock(&mctz->lock); 477 } 478 } 479} 480 481static void mem_cgroup_remove_from_trees(struct mem_cgroup *mem) 482{ 483 int node, zone; 484 struct mem_cgroup_per_zone *mz; 485 struct mem_cgroup_tree_per_zone *mctz; 486 487 for_each_node_state(node, N_POSSIBLE) { 488 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 489 mz = mem_cgroup_zoneinfo(mem, node, zone); 490 mctz = soft_limit_tree_node_zone(node, zone); 491 mem_cgroup_remove_exceeded(mem, mz, mctz); 492 } 493 } 494} 495 496static struct mem_cgroup_per_zone * 497__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz) 498{ 499 struct rb_node *rightmost = NULL; 500 struct mem_cgroup_per_zone *mz; 501 502retry: 503 mz = NULL; 504 rightmost = rb_last(&mctz->rb_root); 505 if (!rightmost) 506 goto done; /* Nothing to reclaim from */ 507 508 mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node); 509 /* 510 * Remove the node now but someone else can add it back, 511 * we will to add it back at the end of reclaim to its correct 512 * position in the tree. 513 */ 514 __mem_cgroup_remove_exceeded(mz->mem, mz, mctz); 515 if (!res_counter_soft_limit_excess(&mz->mem->res) || 516 !css_tryget(&mz->mem->css)) 517 goto retry; 518done: 519 return mz; 520} 521 522static struct mem_cgroup_per_zone * 523mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz) 524{ 525 struct mem_cgroup_per_zone *mz; 526 527 spin_lock(&mctz->lock); 528 mz = __mem_cgroup_largest_soft_limit_node(mctz); 529 spin_unlock(&mctz->lock); 530 return mz; 531} 532 533/* 534 * Implementation Note: reading percpu statistics for memcg. 535 * 536 * Both of vmstat[] and percpu_counter has threshold and do periodic 537 * synchronization to implement "quick" read. There are trade-off between 538 * reading cost and precision of value. Then, we may have a chance to implement 539 * a periodic synchronizion of counter in memcg's counter. 540 * 541 * But this _read() function is used for user interface now. The user accounts 542 * memory usage by memory cgroup and he _always_ requires exact value because 543 * he accounts memory. Even if we provide quick-and-fuzzy read, we always 544 * have to visit all online cpus and make sum. So, for now, unnecessary 545 * synchronization is not implemented. (just implemented for cpu hotplug) 546 * 547 * If there are kernel internal actions which can make use of some not-exact 548 * value, and reading all cpu value can be performance bottleneck in some 549 * common workload, threashold and synchonization as vmstat[] should be 550 * implemented. 551 */ 552static s64 mem_cgroup_read_stat(struct mem_cgroup *mem, 553 enum mem_cgroup_stat_index idx) 554{ 555 int cpu; 556 s64 val = 0; 557 558 get_online_cpus(); 559 for_each_online_cpu(cpu) 560 val += per_cpu(mem->stat->count[idx], cpu); 561#ifdef CONFIG_HOTPLUG_CPU 562 spin_lock(&mem->pcp_counter_lock); 563 val += mem->nocpu_base.count[idx]; 564 spin_unlock(&mem->pcp_counter_lock); 565#endif 566 put_online_cpus(); 567 return val; 568} 569 570static s64 mem_cgroup_local_usage(struct mem_cgroup *mem) 571{ 572 s64 ret; 573 574 ret = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS); 575 ret += mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE); 576 return ret; 577} 578 579static void mem_cgroup_swap_statistics(struct mem_cgroup *mem, 580 bool charge) 581{ 582 int val = (charge) ? 1 : -1; 583 this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_SWAPOUT], val); 584} 585 586static void mem_cgroup_charge_statistics(struct mem_cgroup *mem, 587 bool file, int nr_pages) 588{ 589 preempt_disable(); 590 591 if (file) 592 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_CACHE], nr_pages); 593 else 594 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_RSS], nr_pages); 595 596 /* pagein of a big page is an event. So, ignore page size */ 597 if (nr_pages > 0) 598 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGIN_COUNT]); 599 else { 600 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGOUT_COUNT]); 601 nr_pages = -nr_pages; /* for event */ 602 } 603 604 __this_cpu_add(mem->stat->count[MEM_CGROUP_EVENTS], nr_pages); 605 606 preempt_enable(); 607} 608 609static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem, 610 enum lru_list idx) 611{ 612 int nid, zid; 613 struct mem_cgroup_per_zone *mz; 614 u64 total = 0; 615 616 for_each_online_node(nid) 617 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 618 mz = mem_cgroup_zoneinfo(mem, nid, zid); 619 total += MEM_CGROUP_ZSTAT(mz, idx); 620 } 621 return total; 622} 623 624static bool __memcg_event_check(struct mem_cgroup *mem, int event_mask_shift) 625{ 626 s64 val; 627 628 val = this_cpu_read(mem->stat->count[MEM_CGROUP_EVENTS]); 629 630 return !(val & ((1 << event_mask_shift) - 1)); 631} 632 633/* 634 * Check events in order. 635 * 636 */ 637static void memcg_check_events(struct mem_cgroup *mem, struct page *page) 638{ 639 /* threshold event is triggered in finer grain than soft limit */ 640 if (unlikely(__memcg_event_check(mem, THRESHOLDS_EVENTS_THRESH))) { 641 mem_cgroup_threshold(mem); 642 if (unlikely(__memcg_event_check(mem, SOFTLIMIT_EVENTS_THRESH))) 643 mem_cgroup_update_tree(mem, page); 644 } 645} 646 647static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont) 648{ 649 return container_of(cgroup_subsys_state(cont, 650 mem_cgroup_subsys_id), struct mem_cgroup, 651 css); 652} 653 654struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) 655{ 656 /* 657 * mm_update_next_owner() may clear mm->owner to NULL 658 * if it races with swapoff, page migration, etc. 659 * So this can be called with p == NULL. 660 */ 661 if (unlikely(!p)) 662 return NULL; 663 664 return container_of(task_subsys_state(p, mem_cgroup_subsys_id), 665 struct mem_cgroup, css); 666} 667 668static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm) 669{ 670 struct mem_cgroup *mem = NULL; 671 672 if (!mm) 673 return NULL; 674 /* 675 * Because we have no locks, mm->owner's may be being moved to other 676 * cgroup. We use css_tryget() here even if this looks 677 * pessimistic (rather than adding locks here). 678 */ 679 rcu_read_lock(); 680 do { 681 mem = mem_cgroup_from_task(rcu_dereference(mm->owner)); 682 if (unlikely(!mem)) 683 break; 684 } while (!css_tryget(&mem->css)); 685 rcu_read_unlock(); 686 return mem; 687} 688 689/* The caller has to guarantee "mem" exists before calling this */ 690static struct mem_cgroup *mem_cgroup_start_loop(struct mem_cgroup *mem) 691{ 692 struct cgroup_subsys_state *css; 693 int found; 694 695 if (!mem) /* ROOT cgroup has the smallest ID */ 696 return root_mem_cgroup; /*css_put/get against root is ignored*/ 697 if (!mem->use_hierarchy) { 698 if (css_tryget(&mem->css)) 699 return mem; 700 return NULL; 701 } 702 rcu_read_lock(); 703 /* 704 * searching a memory cgroup which has the smallest ID under given 705 * ROOT cgroup. (ID >= 1) 706 */ 707 css = css_get_next(&mem_cgroup_subsys, 1, &mem->css, &found); 708 if (css && css_tryget(css)) 709 mem = container_of(css, struct mem_cgroup, css); 710 else 711 mem = NULL; 712 rcu_read_unlock(); 713 return mem; 714} 715 716static struct mem_cgroup *mem_cgroup_get_next(struct mem_cgroup *iter, 717 struct mem_cgroup *root, 718 bool cond) 719{ 720 int nextid = css_id(&iter->css) + 1; 721 int found; 722 int hierarchy_used; 723 struct cgroup_subsys_state *css; 724 725 hierarchy_used = iter->use_hierarchy; 726 727 css_put(&iter->css); 728 /* If no ROOT, walk all, ignore hierarchy */ 729 if (!cond || (root && !hierarchy_used)) 730 return NULL; 731 732 if (!root) 733 root = root_mem_cgroup; 734 735 do { 736 iter = NULL; 737 rcu_read_lock(); 738 739 css = css_get_next(&mem_cgroup_subsys, nextid, 740 &root->css, &found); 741 if (css && css_tryget(css)) 742 iter = container_of(css, struct mem_cgroup, css); 743 rcu_read_unlock(); 744 /* If css is NULL, no more cgroups will be found */ 745 nextid = found + 1; 746 } while (css && !iter); 747 748 return iter; 749} 750/* 751 * for_eacn_mem_cgroup_tree() for visiting all cgroup under tree. Please 752 * be careful that "break" loop is not allowed. We have reference count. 753 * Instead of that modify "cond" to be false and "continue" to exit the loop. 754 */ 755#define for_each_mem_cgroup_tree_cond(iter, root, cond) \ 756 for (iter = mem_cgroup_start_loop(root);\ 757 iter != NULL;\ 758 iter = mem_cgroup_get_next(iter, root, cond)) 759 760#define for_each_mem_cgroup_tree(iter, root) \ 761 for_each_mem_cgroup_tree_cond(iter, root, true) 762 763#define for_each_mem_cgroup_all(iter) \ 764 for_each_mem_cgroup_tree_cond(iter, NULL, true) 765 766 767static inline bool mem_cgroup_is_root(struct mem_cgroup *mem) 768{ 769 return (mem == root_mem_cgroup); 770} 771 772/* 773 * Following LRU functions are allowed to be used without PCG_LOCK. 774 * Operations are called by routine of global LRU independently from memcg. 775 * What we have to take care of here is validness of pc->mem_cgroup. 776 * 777 * Changes to pc->mem_cgroup happens when 778 * 1. charge 779 * 2. moving account 780 * In typical case, "charge" is done before add-to-lru. Exception is SwapCache. 781 * It is added to LRU before charge. 782 * If PCG_USED bit is not set, page_cgroup is not added to this private LRU. 783 * When moving account, the page is not on LRU. It's isolated. 784 */ 785 786void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru) 787{ 788 struct page_cgroup *pc; 789 struct mem_cgroup_per_zone *mz; 790 791 if (mem_cgroup_disabled()) 792 return; 793 pc = lookup_page_cgroup(page); 794 /* can happen while we handle swapcache. */ 795 if (!TestClearPageCgroupAcctLRU(pc)) 796 return; 797 VM_BUG_ON(!pc->mem_cgroup); 798 /* 799 * We don't check PCG_USED bit. It's cleared when the "page" is finally 800 * removed from global LRU. 801 */ 802 mz = page_cgroup_zoneinfo(pc->mem_cgroup, page); 803 /* huge page split is done under lru_lock. so, we have no races. */ 804 MEM_CGROUP_ZSTAT(mz, lru) -= 1 << compound_order(page); 805 if (mem_cgroup_is_root(pc->mem_cgroup)) 806 return; 807 VM_BUG_ON(list_empty(&pc->lru)); 808 list_del_init(&pc->lru); 809} 810 811void mem_cgroup_del_lru(struct page *page) 812{ 813 mem_cgroup_del_lru_list(page, page_lru(page)); 814} 815 816/* 817 * Writeback is about to end against a page which has been marked for immediate 818 * reclaim. If it still appears to be reclaimable, move it to the tail of the 819 * inactive list. 820 */ 821void mem_cgroup_rotate_reclaimable_page(struct page *page) 822{ 823 struct mem_cgroup_per_zone *mz; 824 struct page_cgroup *pc; 825 enum lru_list lru = page_lru(page); 826 827 if (mem_cgroup_disabled()) 828 return; 829 830 pc = lookup_page_cgroup(page); 831 /* unused or root page is not rotated. */ 832 if (!PageCgroupUsed(pc)) 833 return; 834 /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */ 835 smp_rmb(); 836 if (mem_cgroup_is_root(pc->mem_cgroup)) 837 return; 838 mz = page_cgroup_zoneinfo(pc->mem_cgroup, page); 839 list_move_tail(&pc->lru, &mz->lists[lru]); 840} 841 842void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru) 843{ 844 struct mem_cgroup_per_zone *mz; 845 struct page_cgroup *pc; 846 847 if (mem_cgroup_disabled()) 848 return; 849 850 pc = lookup_page_cgroup(page); 851 /* unused or root page is not rotated. */ 852 if (!PageCgroupUsed(pc)) 853 return; 854 /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */ 855 smp_rmb(); 856 if (mem_cgroup_is_root(pc->mem_cgroup)) 857 return; 858 mz = page_cgroup_zoneinfo(pc->mem_cgroup, page); 859 list_move(&pc->lru, &mz->lists[lru]); 860} 861 862void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru) 863{ 864 struct page_cgroup *pc; 865 struct mem_cgroup_per_zone *mz; 866 867 if (mem_cgroup_disabled()) 868 return; 869 pc = lookup_page_cgroup(page); 870 VM_BUG_ON(PageCgroupAcctLRU(pc)); 871 if (!PageCgroupUsed(pc)) 872 return; 873 /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */ 874 smp_rmb(); 875 mz = page_cgroup_zoneinfo(pc->mem_cgroup, page); 876 /* huge page split is done under lru_lock. so, we have no races. */ 877 MEM_CGROUP_ZSTAT(mz, lru) += 1 << compound_order(page); 878 SetPageCgroupAcctLRU(pc); 879 if (mem_cgroup_is_root(pc->mem_cgroup)) 880 return; 881 list_add(&pc->lru, &mz->lists[lru]); 882} 883 884/* 885 * At handling SwapCache, pc->mem_cgroup may be changed while it's linked to 886 * lru because the page may.be reused after it's fully uncharged (because of 887 * SwapCache behavior).To handle that, unlink page_cgroup from LRU when charge 888 * it again. This function is only used to charge SwapCache. It's done under 889 * lock_page and expected that zone->lru_lock is never held. 890 */ 891static void mem_cgroup_lru_del_before_commit_swapcache(struct page *page) 892{ 893 unsigned long flags; 894 struct zone *zone = page_zone(page); 895 struct page_cgroup *pc = lookup_page_cgroup(page); 896 897 spin_lock_irqsave(&zone->lru_lock, flags); 898 /* 899 * Forget old LRU when this page_cgroup is *not* used. This Used bit 900 * is guarded by lock_page() because the page is SwapCache. 901 */ 902 if (!PageCgroupUsed(pc)) 903 mem_cgroup_del_lru_list(page, page_lru(page)); 904 spin_unlock_irqrestore(&zone->lru_lock, flags); 905} 906 907static void mem_cgroup_lru_add_after_commit_swapcache(struct page *page) 908{ 909 unsigned long flags; 910 struct zone *zone = page_zone(page); 911 struct page_cgroup *pc = lookup_page_cgroup(page); 912 913 spin_lock_irqsave(&zone->lru_lock, flags); 914 /* link when the page is linked to LRU but page_cgroup isn't */ 915 if (PageLRU(page) && !PageCgroupAcctLRU(pc)) 916 mem_cgroup_add_lru_list(page, page_lru(page)); 917 spin_unlock_irqrestore(&zone->lru_lock, flags); 918} 919 920 921void mem_cgroup_move_lists(struct page *page, 922 enum lru_list from, enum lru_list to) 923{ 924 if (mem_cgroup_disabled()) 925 return; 926 mem_cgroup_del_lru_list(page, from); 927 mem_cgroup_add_lru_list(page, to); 928} 929 930int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem) 931{ 932 int ret; 933 struct mem_cgroup *curr = NULL; 934 struct task_struct *p; 935 936 p = find_lock_task_mm(task); 937 if (!p) 938 return 0; 939 curr = try_get_mem_cgroup_from_mm(p->mm); 940 task_unlock(p); 941 if (!curr) 942 return 0; 943 /* 944 * We should check use_hierarchy of "mem" not "curr". Because checking 945 * use_hierarchy of "curr" here make this function true if hierarchy is 946 * enabled in "curr" and "curr" is a child of "mem" in *cgroup* 947 * hierarchy(even if use_hierarchy is disabled in "mem"). 948 */ 949 if (mem->use_hierarchy) 950 ret = css_is_ancestor(&curr->css, &mem->css); 951 else 952 ret = (curr == mem); 953 css_put(&curr->css); 954 return ret; 955} 956 957static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_pages) 958{ 959 unsigned long active; 960 unsigned long inactive; 961 unsigned long gb; 962 unsigned long inactive_ratio; 963 964 inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON); 965 active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON); 966 967 gb = (inactive + active) >> (30 - PAGE_SHIFT); 968 if (gb) 969 inactive_ratio = int_sqrt(10 * gb); 970 else 971 inactive_ratio = 1; 972 973 if (present_pages) { 974 present_pages[0] = inactive; 975 present_pages[1] = active; 976 } 977 978 return inactive_ratio; 979} 980 981int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg) 982{ 983 unsigned long active; 984 unsigned long inactive; 985 unsigned long present_pages[2]; 986 unsigned long inactive_ratio; 987 988 inactive_ratio = calc_inactive_ratio(memcg, present_pages); 989 990 inactive = present_pages[0]; 991 active = present_pages[1]; 992 993 if (inactive * inactive_ratio < active) 994 return 1; 995 996 return 0; 997} 998 999int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg) 1000{ 1001 unsigned long active; 1002 unsigned long inactive; 1003 1004 inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE); 1005 active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE); 1006 1007 return (active > inactive); 1008} 1009 1010unsigned long mem_cgroup_zone_nr_pages(struct mem_cgroup *memcg, 1011 struct zone *zone, 1012 enum lru_list lru) 1013{ 1014 int nid = zone_to_nid(zone); 1015 int zid = zone_idx(zone); 1016 struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid); 1017 1018 return MEM_CGROUP_ZSTAT(mz, lru); 1019} 1020 1021struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg, 1022 struct zone *zone) 1023{ 1024 int nid = zone_to_nid(zone); 1025 int zid = zone_idx(zone); 1026 struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid); 1027 1028 return &mz->reclaim_stat; 1029} 1030 1031struct zone_reclaim_stat * 1032mem_cgroup_get_reclaim_stat_from_page(struct page *page) 1033{ 1034 struct page_cgroup *pc; 1035 struct mem_cgroup_per_zone *mz; 1036 1037 if (mem_cgroup_disabled()) 1038 return NULL; 1039 1040 pc = lookup_page_cgroup(page); 1041 if (!PageCgroupUsed(pc)) 1042 return NULL; 1043 /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */ 1044 smp_rmb(); 1045 mz = page_cgroup_zoneinfo(pc->mem_cgroup, page); 1046 if (!mz) 1047 return NULL; 1048 1049 return &mz->reclaim_stat; 1050} 1051 1052unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan, 1053 struct list_head *dst, 1054 unsigned long *scanned, int order, 1055 int mode, struct zone *z, 1056 struct mem_cgroup *mem_cont, 1057 int active, int file) 1058{ 1059 unsigned long nr_taken = 0; 1060 struct page *page; 1061 unsigned long scan; 1062 LIST_HEAD(pc_list); 1063 struct list_head *src; 1064 struct page_cgroup *pc, *tmp; 1065 int nid = zone_to_nid(z); 1066 int zid = zone_idx(z); 1067 struct mem_cgroup_per_zone *mz; 1068 int lru = LRU_FILE * file + active; 1069 int ret; 1070 1071 BUG_ON(!mem_cont); 1072 mz = mem_cgroup_zoneinfo(mem_cont, nid, zid); 1073 src = &mz->lists[lru]; 1074 1075 scan = 0; 1076 list_for_each_entry_safe_reverse(pc, tmp, src, lru) { 1077 if (scan >= nr_to_scan) 1078 break; 1079 1080 if (unlikely(!PageCgroupUsed(pc))) 1081 continue; 1082 1083 page = pc->page; 1084 1085 if (unlikely(!PageLRU(page))) 1086 continue; 1087 1088 scan++; 1089 ret = __isolate_lru_page(page, mode, file); 1090 switch (ret) { 1091 case 0: 1092 list_move(&page->lru, dst); 1093 mem_cgroup_del_lru(page); 1094 nr_taken += hpage_nr_pages(page); 1095 break; 1096 case -EBUSY: 1097 /* we don't affect global LRU but rotate in our LRU */ 1098 mem_cgroup_rotate_lru_list(page, page_lru(page)); 1099 break; 1100 default: 1101 break; 1102 } 1103 } 1104 1105 *scanned = scan; 1106 1107 trace_mm_vmscan_memcg_isolate(0, nr_to_scan, scan, nr_taken, 1108 0, 0, 0, mode); 1109 1110 return nr_taken; 1111} 1112 1113#define mem_cgroup_from_res_counter(counter, member) \ 1114 container_of(counter, struct mem_cgroup, member) 1115 1116/** 1117 * mem_cgroup_margin - calculate chargeable space of a memory cgroup 1118 * @mem: the memory cgroup 1119 * 1120 * Returns the maximum amount of memory @mem can be charged with, in 1121 * bytes. 1122 */ 1123static unsigned long long mem_cgroup_margin(struct mem_cgroup *mem) 1124{ 1125 unsigned long long margin; 1126 1127 margin = res_counter_margin(&mem->res); 1128 if (do_swap_account) 1129 margin = min(margin, res_counter_margin(&mem->memsw)); 1130 return margin; 1131} 1132 1133static unsigned int get_swappiness(struct mem_cgroup *memcg) 1134{ 1135 struct cgroup *cgrp = memcg->css.cgroup; 1136 unsigned int swappiness; 1137 1138 /* root ? */ 1139 if (cgrp->parent == NULL) 1140 return vm_swappiness; 1141 1142 spin_lock(&memcg->reclaim_param_lock); 1143 swappiness = memcg->swappiness; 1144 spin_unlock(&memcg->reclaim_param_lock); 1145 1146 return swappiness; 1147} 1148 1149static void mem_cgroup_start_move(struct mem_cgroup *mem) 1150{ 1151 int cpu; 1152 1153 get_online_cpus(); 1154 spin_lock(&mem->pcp_counter_lock); 1155 for_each_online_cpu(cpu) 1156 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) += 1; 1157 mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] += 1; 1158 spin_unlock(&mem->pcp_counter_lock); 1159 put_online_cpus(); 1160 1161 synchronize_rcu(); 1162} 1163 1164static void mem_cgroup_end_move(struct mem_cgroup *mem) 1165{ 1166 int cpu; 1167 1168 if (!mem) 1169 return; 1170 get_online_cpus(); 1171 spin_lock(&mem->pcp_counter_lock); 1172 for_each_online_cpu(cpu) 1173 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) -= 1; 1174 mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] -= 1; 1175 spin_unlock(&mem->pcp_counter_lock); 1176 put_online_cpus(); 1177} 1178/* 1179 * 2 routines for checking "mem" is under move_account() or not. 1180 * 1181 * mem_cgroup_stealed() - checking a cgroup is mc.from or not. This is used 1182 * for avoiding race in accounting. If true, 1183 * pc->mem_cgroup may be overwritten. 1184 * 1185 * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or 1186 * under hierarchy of moving cgroups. This is for 1187 * waiting at hith-memory prressure caused by "move". 1188 */ 1189 1190static bool mem_cgroup_stealed(struct mem_cgroup *mem) 1191{ 1192 VM_BUG_ON(!rcu_read_lock_held()); 1193 return this_cpu_read(mem->stat->count[MEM_CGROUP_ON_MOVE]) > 0; 1194} 1195 1196static bool mem_cgroup_under_move(struct mem_cgroup *mem) 1197{ 1198 struct mem_cgroup *from; 1199 struct mem_cgroup *to; 1200 bool ret = false; 1201 /* 1202 * Unlike task_move routines, we access mc.to, mc.from not under 1203 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead. 1204 */ 1205 spin_lock(&mc.lock); 1206 from = mc.from; 1207 to = mc.to; 1208 if (!from) 1209 goto unlock; 1210 if (from == mem || to == mem 1211 || (mem->use_hierarchy && css_is_ancestor(&from->css, &mem->css)) 1212 || (mem->use_hierarchy && css_is_ancestor(&to->css, &mem->css))) 1213 ret = true; 1214unlock: 1215 spin_unlock(&mc.lock); 1216 return ret; 1217} 1218 1219static bool mem_cgroup_wait_acct_move(struct mem_cgroup *mem) 1220{ 1221 if (mc.moving_task && current != mc.moving_task) { 1222 if (mem_cgroup_under_move(mem)) { 1223 DEFINE_WAIT(wait); 1224 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE); 1225 /* moving charge context might have finished. */ 1226 if (mc.moving_task) 1227 schedule(); 1228 finish_wait(&mc.waitq, &wait); 1229 return true; 1230 } 1231 } 1232 return false; 1233} 1234 1235/** 1236 * mem_cgroup_print_oom_info: Called from OOM with tasklist_lock held in read mode. 1237 * @memcg: The memory cgroup that went over limit 1238 * @p: Task that is going to be killed 1239 * 1240 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is 1241 * enabled 1242 */ 1243void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p) 1244{ 1245 struct cgroup *task_cgrp; 1246 struct cgroup *mem_cgrp; 1247 /* 1248 * Need a buffer in BSS, can't rely on allocations. The code relies 1249 * on the assumption that OOM is serialized for memory controller. 1250 * If this assumption is broken, revisit this code. 1251 */ 1252 static char memcg_name[PATH_MAX]; 1253 int ret; 1254 1255 if (!memcg || !p) 1256 return; 1257 1258 1259 rcu_read_lock(); 1260 1261 mem_cgrp = memcg->css.cgroup; 1262 task_cgrp = task_cgroup(p, mem_cgroup_subsys_id); 1263 1264 ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX); 1265 if (ret < 0) { 1266 /* 1267 * Unfortunately, we are unable to convert to a useful name 1268 * But we'll still print out the usage information 1269 */ 1270 rcu_read_unlock(); 1271 goto done; 1272 } 1273 rcu_read_unlock(); 1274 1275 printk(KERN_INFO "Task in %s killed", memcg_name); 1276 1277 rcu_read_lock(); 1278 ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX); 1279 if (ret < 0) { 1280 rcu_read_unlock(); 1281 goto done; 1282 } 1283 rcu_read_unlock(); 1284 1285 /* 1286 * Continues from above, so we don't need an KERN_ level 1287 */ 1288 printk(KERN_CONT " as a result of limit of %s\n", memcg_name); 1289done: 1290 1291 printk(KERN_INFO "memory: usage %llukB, limit %llukB, failcnt %llu\n", 1292 res_counter_read_u64(&memcg->res, RES_USAGE) >> 10, 1293 res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10, 1294 res_counter_read_u64(&memcg->res, RES_FAILCNT)); 1295 printk(KERN_INFO "memory+swap: usage %llukB, limit %llukB, " 1296 "failcnt %llu\n", 1297 res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10, 1298 res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10, 1299 res_counter_read_u64(&memcg->memsw, RES_FAILCNT)); 1300} 1301 1302/* 1303 * This function returns the number of memcg under hierarchy tree. Returns 1304 * 1(self count) if no children. 1305 */ 1306static int mem_cgroup_count_children(struct mem_cgroup *mem) 1307{ 1308 int num = 0; 1309 struct mem_cgroup *iter; 1310 1311 for_each_mem_cgroup_tree(iter, mem) 1312 num++; 1313 return num; 1314} 1315 1316/* 1317 * Return the memory (and swap, if configured) limit for a memcg. 1318 */ 1319u64 mem_cgroup_get_limit(struct mem_cgroup *memcg) 1320{ 1321 u64 limit; 1322 u64 memsw; 1323 1324 limit = res_counter_read_u64(&memcg->res, RES_LIMIT); 1325 limit += total_swap_pages << PAGE_SHIFT; 1326 1327 memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 1328 /* 1329 * If memsw is finite and limits the amount of swap space available 1330 * to this memcg, return that limit. 1331 */ 1332 return min(limit, memsw); 1333} 1334 1335/* 1336 * Visit the first child (need not be the first child as per the ordering 1337 * of the cgroup list, since we track last_scanned_child) of @mem and use 1338 * that to reclaim free pages from. 1339 */ 1340static struct mem_cgroup * 1341mem_cgroup_select_victim(struct mem_cgroup *root_mem) 1342{ 1343 struct mem_cgroup *ret = NULL; 1344 struct cgroup_subsys_state *css; 1345 int nextid, found; 1346 1347 if (!root_mem->use_hierarchy) { 1348 css_get(&root_mem->css); 1349 ret = root_mem; 1350 } 1351 1352 while (!ret) { 1353 rcu_read_lock(); 1354 nextid = root_mem->last_scanned_child + 1; 1355 css = css_get_next(&mem_cgroup_subsys, nextid, &root_mem->css, 1356 &found); 1357 if (css && css_tryget(css)) 1358 ret = container_of(css, struct mem_cgroup, css); 1359 1360 rcu_read_unlock(); 1361 /* Updates scanning parameter */ 1362 spin_lock(&root_mem->reclaim_param_lock); 1363 if (!css) { 1364 /* this means start scan from ID:1 */ 1365 root_mem->last_scanned_child = 0; 1366 } else 1367 root_mem->last_scanned_child = found; 1368 spin_unlock(&root_mem->reclaim_param_lock); 1369 } 1370 1371 return ret; 1372} 1373 1374/* 1375 * Scan the hierarchy if needed to reclaim memory. We remember the last child 1376 * we reclaimed from, so that we don't end up penalizing one child extensively 1377 * based on its position in the children list. 1378 * 1379 * root_mem is the original ancestor that we've been reclaim from. 1380 * 1381 * We give up and return to the caller when we visit root_mem twice. 1382 * (other groups can be removed while we're walking....) 1383 * 1384 * If shrink==true, for avoiding to free too much, this returns immedieately. 1385 */ 1386static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem, 1387 struct zone *zone, 1388 gfp_t gfp_mask, 1389 unsigned long reclaim_options) 1390{ 1391 struct mem_cgroup *victim; 1392 int ret, total = 0; 1393 int loop = 0; 1394 bool noswap = reclaim_options & MEM_CGROUP_RECLAIM_NOSWAP; 1395 bool shrink = reclaim_options & MEM_CGROUP_RECLAIM_SHRINK; 1396 bool check_soft = reclaim_options & MEM_CGROUP_RECLAIM_SOFT; 1397 unsigned long excess; 1398 1399 excess = res_counter_soft_limit_excess(&root_mem->res) >> PAGE_SHIFT; 1400 1401 /* If memsw_is_minimum==1, swap-out is of-no-use. */ 1402 if (root_mem->memsw_is_minimum) 1403 noswap = true; 1404 1405 while (1) { 1406 victim = mem_cgroup_select_victim(root_mem); 1407 if (victim == root_mem) { 1408 loop++; 1409 if (loop >= 1) 1410 drain_all_stock_async(); 1411 if (loop >= 2) { 1412 /* 1413 * If we have not been able to reclaim 1414 * anything, it might because there are 1415 * no reclaimable pages under this hierarchy 1416 */ 1417 if (!check_soft || !total) { 1418 css_put(&victim->css); 1419 break; 1420 } 1421 /* 1422 * We want to do more targetted reclaim. 1423 * excess >> 2 is not to excessive so as to 1424 * reclaim too much, nor too less that we keep 1425 * coming back to reclaim from this cgroup 1426 */ 1427 if (total >= (excess >> 2) || 1428 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) { 1429 css_put(&victim->css); 1430 break; 1431 } 1432 } 1433 } 1434 if (!mem_cgroup_local_usage(victim)) { 1435 /* this cgroup's local usage == 0 */ 1436 css_put(&victim->css); 1437 continue; 1438 } 1439 /* we use swappiness of local cgroup */ 1440 if (check_soft) 1441 ret = mem_cgroup_shrink_node_zone(victim, gfp_mask, 1442 noswap, get_swappiness(victim), zone); 1443 else 1444 ret = try_to_free_mem_cgroup_pages(victim, gfp_mask, 1445 noswap, get_swappiness(victim)); 1446 css_put(&victim->css); 1447 /* 1448 * At shrinking usage, we can't check we should stop here or 1449 * reclaim more. It's depends on callers. last_scanned_child 1450 * will work enough for keeping fairness under tree. 1451 */ 1452 if (shrink) 1453 return ret; 1454 total += ret; 1455 if (check_soft) { 1456 if (!res_counter_soft_limit_excess(&root_mem->res)) 1457 return total; 1458 } else if (mem_cgroup_margin(root_mem)) 1459 return 1 + total; 1460 } 1461 return total; 1462} 1463 1464/* 1465 * Check OOM-Killer is already running under our hierarchy. 1466 * If someone is running, return false. 1467 */ 1468static bool mem_cgroup_oom_lock(struct mem_cgroup *mem) 1469{ 1470 int x, lock_count = 0; 1471 struct mem_cgroup *iter; 1472 1473 for_each_mem_cgroup_tree(iter, mem) { 1474 x = atomic_inc_return(&iter->oom_lock); 1475 lock_count = max(x, lock_count); 1476 } 1477 1478 if (lock_count == 1) 1479 return true; 1480 return false; 1481} 1482 1483static int mem_cgroup_oom_unlock(struct mem_cgroup *mem) 1484{ 1485 struct mem_cgroup *iter; 1486 1487 /* 1488 * When a new child is created while the hierarchy is under oom, 1489 * mem_cgroup_oom_lock() may not be called. We have to use 1490 * atomic_add_unless() here. 1491 */ 1492 for_each_mem_cgroup_tree(iter, mem) 1493 atomic_add_unless(&iter->oom_lock, -1, 0); 1494 return 0; 1495} 1496 1497 1498static DEFINE_MUTEX(memcg_oom_mutex); 1499static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq); 1500 1501struct oom_wait_info { 1502 struct mem_cgroup *mem; 1503 wait_queue_t wait; 1504}; 1505 1506static int memcg_oom_wake_function(wait_queue_t *wait, 1507 unsigned mode, int sync, void *arg) 1508{ 1509 struct mem_cgroup *wake_mem = (struct mem_cgroup *)arg; 1510 struct oom_wait_info *oom_wait_info; 1511 1512 oom_wait_info = container_of(wait, struct oom_wait_info, wait); 1513 1514 if (oom_wait_info->mem == wake_mem) 1515 goto wakeup; 1516 /* if no hierarchy, no match */ 1517 if (!oom_wait_info->mem->use_hierarchy || !wake_mem->use_hierarchy) 1518 return 0; 1519 /* 1520 * Both of oom_wait_info->mem and wake_mem are stable under us. 1521 * Then we can use css_is_ancestor without taking care of RCU. 1522 */ 1523 if (!css_is_ancestor(&oom_wait_info->mem->css, &wake_mem->css) && 1524 !css_is_ancestor(&wake_mem->css, &oom_wait_info->mem->css)) 1525 return 0; 1526 1527wakeup: 1528 return autoremove_wake_function(wait, mode, sync, arg); 1529} 1530 1531static void memcg_wakeup_oom(struct mem_cgroup *mem) 1532{ 1533 /* for filtering, pass "mem" as argument. */ 1534 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, mem); 1535} 1536 1537static void memcg_oom_recover(struct mem_cgroup *mem) 1538{ 1539 if (mem && atomic_read(&mem->oom_lock)) 1540 memcg_wakeup_oom(mem); 1541} 1542 1543/* 1544 * try to call OOM killer. returns false if we should exit memory-reclaim loop. 1545 */ 1546bool mem_cgroup_handle_oom(struct mem_cgroup *mem, gfp_t mask) 1547{ 1548 struct oom_wait_info owait; 1549 bool locked, need_to_kill; 1550 1551 owait.mem = mem; 1552 owait.wait.flags = 0; 1553 owait.wait.func = memcg_oom_wake_function; 1554 owait.wait.private = current; 1555 INIT_LIST_HEAD(&owait.wait.task_list); 1556 need_to_kill = true; 1557 /* At first, try to OOM lock hierarchy under mem.*/ 1558 mutex_lock(&memcg_oom_mutex); 1559 locked = mem_cgroup_oom_lock(mem); 1560 /* 1561 * Even if signal_pending(), we can't quit charge() loop without 1562 * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL 1563 * under OOM is always welcomed, use TASK_KILLABLE here. 1564 */ 1565 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE); 1566 if (!locked || mem->oom_kill_disable) 1567 need_to_kill = false; 1568 if (locked) 1569 mem_cgroup_oom_notify(mem); 1570 mutex_unlock(&memcg_oom_mutex); 1571 1572 if (need_to_kill) { 1573 finish_wait(&memcg_oom_waitq, &owait.wait); 1574 mem_cgroup_out_of_memory(mem, mask); 1575 } else { 1576 schedule(); 1577 finish_wait(&memcg_oom_waitq, &owait.wait); 1578 } 1579 mutex_lock(&memcg_oom_mutex); 1580 mem_cgroup_oom_unlock(mem); 1581 memcg_wakeup_oom(mem); 1582 mutex_unlock(&memcg_oom_mutex); 1583 1584 if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current)) 1585 return false; 1586 /* Give chance to dying process */ 1587 schedule_timeout(1); 1588 return true; 1589} 1590 1591/* 1592 * Currently used to update mapped file statistics, but the routine can be 1593 * generalized to update other statistics as well. 1594 * 1595 * Notes: Race condition 1596 * 1597 * We usually use page_cgroup_lock() for accessing page_cgroup member but 1598 * it tends to be costly. But considering some conditions, we doesn't need 1599 * to do so _always_. 1600 * 1601 * Considering "charge", lock_page_cgroup() is not required because all 1602 * file-stat operations happen after a page is attached to radix-tree. There 1603 * are no race with "charge". 1604 * 1605 * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup 1606 * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even 1607 * if there are race with "uncharge". Statistics itself is properly handled 1608 * by flags. 1609 * 1610 * Considering "move", this is an only case we see a race. To make the race 1611 * small, we check MEM_CGROUP_ON_MOVE percpu value and detect there are 1612 * possibility of race condition. If there is, we take a lock. 1613 */ 1614 1615void mem_cgroup_update_page_stat(struct page *page, 1616 enum mem_cgroup_page_stat_item idx, int val) 1617{ 1618 struct mem_cgroup *mem; 1619 struct page_cgroup *pc = lookup_page_cgroup(page); 1620 bool need_unlock = false; 1621 unsigned long uninitialized_var(flags); 1622 1623 if (unlikely(!pc)) 1624 return; 1625 1626 rcu_read_lock(); 1627 mem = pc->mem_cgroup; 1628 if (unlikely(!mem || !PageCgroupUsed(pc))) 1629 goto out; 1630 /* pc->mem_cgroup is unstable ? */ 1631 if (unlikely(mem_cgroup_stealed(mem)) || PageTransHuge(page)) { 1632 /* take a lock against to access pc->mem_cgroup */ 1633 move_lock_page_cgroup(pc, &flags); 1634 need_unlock = true; 1635 mem = pc->mem_cgroup; 1636 if (!mem || !PageCgroupUsed(pc)) 1637 goto out; 1638 } 1639 1640 switch (idx) { 1641 case MEMCG_NR_FILE_MAPPED: 1642 if (val > 0) 1643 SetPageCgroupFileMapped(pc); 1644 else if (!page_mapped(page)) 1645 ClearPageCgroupFileMapped(pc); 1646 idx = MEM_CGROUP_STAT_FILE_MAPPED; 1647 break; 1648 default: 1649 BUG(); 1650 } 1651 1652 this_cpu_add(mem->stat->count[idx], val); 1653 1654out: 1655 if (unlikely(need_unlock)) 1656 move_unlock_page_cgroup(pc, &flags); 1657 rcu_read_unlock(); 1658 return; 1659} 1660EXPORT_SYMBOL(mem_cgroup_update_page_stat); 1661 1662/* 1663 * size of first charge trial. "32" comes from vmscan.c's magic value. 1664 * TODO: maybe necessary to use big numbers in big irons. 1665 */ 1666#define CHARGE_SIZE (32 * PAGE_SIZE) 1667struct memcg_stock_pcp { 1668 struct mem_cgroup *cached; /* this never be root cgroup */ 1669 int charge; 1670 struct work_struct work; 1671}; 1672static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock); 1673static atomic_t memcg_drain_count; 1674 1675/* 1676 * Try to consume stocked charge on this cpu. If success, PAGE_SIZE is consumed 1677 * from local stock and true is returned. If the stock is 0 or charges from a 1678 * cgroup which is not current target, returns false. This stock will be 1679 * refilled. 1680 */ 1681static bool consume_stock(struct mem_cgroup *mem) 1682{ 1683 struct memcg_stock_pcp *stock; 1684 bool ret = true; 1685 1686 stock = &get_cpu_var(memcg_stock); 1687 if (mem == stock->cached && stock->charge) 1688 stock->charge -= PAGE_SIZE; 1689 else /* need to call res_counter_charge */ 1690 ret = false; 1691 put_cpu_var(memcg_stock); 1692 return ret; 1693} 1694 1695/* 1696 * Returns stocks cached in percpu to res_counter and reset cached information. 1697 */ 1698static void drain_stock(struct memcg_stock_pcp *stock) 1699{ 1700 struct mem_cgroup *old = stock->cached; 1701 1702 if (stock->charge) { 1703 res_counter_uncharge(&old->res, stock->charge); 1704 if (do_swap_account) 1705 res_counter_uncharge(&old->memsw, stock->charge); 1706 } 1707 stock->cached = NULL; 1708 stock->charge = 0; 1709} 1710 1711/* 1712 * This must be called under preempt disabled or must be called by 1713 * a thread which is pinned to local cpu. 1714 */ 1715static void drain_local_stock(struct work_struct *dummy) 1716{ 1717 struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock); 1718 drain_stock(stock); 1719} 1720 1721/* 1722 * Cache charges(val) which is from res_counter, to local per_cpu area. 1723 * This will be consumed by consume_stock() function, later. 1724 */ 1725static void refill_stock(struct mem_cgroup *mem, int val) 1726{ 1727 struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock); 1728 1729 if (stock->cached != mem) { /* reset if necessary */ 1730 drain_stock(stock); 1731 stock->cached = mem; 1732 } 1733 stock->charge += val; 1734 put_cpu_var(memcg_stock); 1735} 1736 1737/* 1738 * Tries to drain stocked charges in other cpus. This function is asynchronous 1739 * and just put a work per cpu for draining localy on each cpu. Caller can 1740 * expects some charges will be back to res_counter later but cannot wait for 1741 * it. 1742 */ 1743static void drain_all_stock_async(void) 1744{ 1745 int cpu; 1746 /* This function is for scheduling "drain" in asynchronous way. 1747 * The result of "drain" is not directly handled by callers. Then, 1748 * if someone is calling drain, we don't have to call drain more. 1749 * Anyway, WORK_STRUCT_PENDING check in queue_work_on() will catch if 1750 * there is a race. We just do loose check here. 1751 */ 1752 if (atomic_read(&memcg_drain_count)) 1753 return; 1754 /* Notify other cpus that system-wide "drain" is running */ 1755 atomic_inc(&memcg_drain_count); 1756 get_online_cpus(); 1757 for_each_online_cpu(cpu) { 1758 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); 1759 schedule_work_on(cpu, &stock->work); 1760 } 1761 put_online_cpus(); 1762 atomic_dec(&memcg_drain_count); 1763 /* We don't wait for flush_work */ 1764} 1765 1766/* This is a synchronous drain interface. */ 1767static void drain_all_stock_sync(void) 1768{ 1769 /* called when force_empty is called */ 1770 atomic_inc(&memcg_drain_count); 1771 schedule_on_each_cpu(drain_local_stock); 1772 atomic_dec(&memcg_drain_count); 1773} 1774 1775/* 1776 * This function drains percpu counter value from DEAD cpu and 1777 * move it to local cpu. Note that this function can be preempted. 1778 */ 1779static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *mem, int cpu) 1780{ 1781 int i; 1782 1783 spin_lock(&mem->pcp_counter_lock); 1784 for (i = 0; i < MEM_CGROUP_STAT_DATA; i++) { 1785 s64 x = per_cpu(mem->stat->count[i], cpu); 1786 1787 per_cpu(mem->stat->count[i], cpu) = 0; 1788 mem->nocpu_base.count[i] += x; 1789 } 1790 /* need to clear ON_MOVE value, works as a kind of lock. */ 1791 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) = 0; 1792 spin_unlock(&mem->pcp_counter_lock); 1793} 1794 1795static void synchronize_mem_cgroup_on_move(struct mem_cgroup *mem, int cpu) 1796{ 1797 int idx = MEM_CGROUP_ON_MOVE; 1798 1799 spin_lock(&mem->pcp_counter_lock); 1800 per_cpu(mem->stat->count[idx], cpu) = mem->nocpu_base.count[idx]; 1801 spin_unlock(&mem->pcp_counter_lock); 1802} 1803 1804static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb, 1805 unsigned long action, 1806 void *hcpu) 1807{ 1808 int cpu = (unsigned long)hcpu; 1809 struct memcg_stock_pcp *stock; 1810 struct mem_cgroup *iter; 1811 1812 if ((action == CPU_ONLINE)) { 1813 for_each_mem_cgroup_all(iter) 1814 synchronize_mem_cgroup_on_move(iter, cpu); 1815 return NOTIFY_OK; 1816 } 1817 1818 if ((action != CPU_DEAD) || action != CPU_DEAD_FROZEN) 1819 return NOTIFY_OK; 1820 1821 for_each_mem_cgroup_all(iter) 1822 mem_cgroup_drain_pcp_counter(iter, cpu); 1823 1824 stock = &per_cpu(memcg_stock, cpu); 1825 drain_stock(stock); 1826 return NOTIFY_OK; 1827} 1828 1829 1830/* See __mem_cgroup_try_charge() for details */ 1831enum { 1832 CHARGE_OK, /* success */ 1833 CHARGE_RETRY, /* need to retry but retry is not bad */ 1834 CHARGE_NOMEM, /* we can't do more. return -ENOMEM */ 1835 CHARGE_WOULDBLOCK, /* GFP_WAIT wasn't set and no enough res. */ 1836 CHARGE_OOM_DIE, /* the current is killed because of OOM */ 1837}; 1838 1839static int __mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask, 1840 int csize, bool oom_check) 1841{ 1842 struct mem_cgroup *mem_over_limit; 1843 struct res_counter *fail_res; 1844 unsigned long flags = 0; 1845 int ret; 1846 1847 ret = res_counter_charge(&mem->res, csize, &fail_res); 1848 1849 if (likely(!ret)) { 1850 if (!do_swap_account) 1851 return CHARGE_OK; 1852 ret = res_counter_charge(&mem->memsw, csize, &fail_res); 1853 if (likely(!ret)) 1854 return CHARGE_OK; 1855 1856 res_counter_uncharge(&mem->res, csize); 1857 mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw); 1858 flags |= MEM_CGROUP_RECLAIM_NOSWAP; 1859 } else 1860 mem_over_limit = mem_cgroup_from_res_counter(fail_res, res); 1861 /* 1862 * csize can be either a huge page (HPAGE_SIZE), a batch of 1863 * regular pages (CHARGE_SIZE), or a single regular page 1864 * (PAGE_SIZE). 1865 * 1866 * Never reclaim on behalf of optional batching, retry with a 1867 * single page instead. 1868 */ 1869 if (csize == CHARGE_SIZE) 1870 return CHARGE_RETRY; 1871 1872 if (!(gfp_mask & __GFP_WAIT)) 1873 return CHARGE_WOULDBLOCK; 1874 1875 ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL, 1876 gfp_mask, flags); 1877 if (mem_cgroup_margin(mem_over_limit) >= csize) 1878 return CHARGE_RETRY; 1879 /* 1880 * Even though the limit is exceeded at this point, reclaim 1881 * may have been able to free some pages. Retry the charge 1882 * before killing the task. 1883 * 1884 * Only for regular pages, though: huge pages are rather 1885 * unlikely to succeed so close to the limit, and we fall back 1886 * to regular pages anyway in case of failure. 1887 */ 1888 if (csize == PAGE_SIZE && ret) 1889 return CHARGE_RETRY; 1890 1891 /* 1892 * At task move, charge accounts can be doubly counted. So, it's 1893 * better to wait until the end of task_move if something is going on. 1894 */ 1895 if (mem_cgroup_wait_acct_move(mem_over_limit)) 1896 return CHARGE_RETRY; 1897 1898 /* If we don't need to call oom-killer at el, return immediately */ 1899 if (!oom_check) 1900 return CHARGE_NOMEM; 1901 /* check OOM */ 1902 if (!mem_cgroup_handle_oom(mem_over_limit, gfp_mask)) 1903 return CHARGE_OOM_DIE; 1904 1905 return CHARGE_RETRY; 1906} 1907 1908/* 1909 * Unlike exported interface, "oom" parameter is added. if oom==true, 1910 * oom-killer can be invoked. 1911 */ 1912static int __mem_cgroup_try_charge(struct mm_struct *mm, 1913 gfp_t gfp_mask, 1914 struct mem_cgroup **memcg, bool oom, 1915 int page_size) 1916{ 1917 int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES; 1918 struct mem_cgroup *mem = NULL; 1919 int ret; 1920 int csize = max(CHARGE_SIZE, (unsigned long) page_size); 1921 1922 /* 1923 * Unlike gloval-vm's OOM-kill, we're not in memory shortage 1924 * in system level. So, allow to go ahead dying process in addition to 1925 * MEMDIE process. 1926 */ 1927 if (unlikely(test_thread_flag(TIF_MEMDIE) 1928 || fatal_signal_pending(current))) 1929 goto bypass; 1930 1931 /* 1932 * We always charge the cgroup the mm_struct belongs to. 1933 * The mm_struct's mem_cgroup changes on task migration if the 1934 * thread group leader migrates. It's possible that mm is not 1935 * set, if so charge the init_mm (happens for pagecache usage). 1936 */ 1937 if (!*memcg && !mm) 1938 goto bypass; 1939again: 1940 if (*memcg) { /* css should be a valid one */ 1941 mem = *memcg; 1942 VM_BUG_ON(css_is_removed(&mem->css)); 1943 if (mem_cgroup_is_root(mem)) 1944 goto done; 1945 if (page_size == PAGE_SIZE && consume_stock(mem)) 1946 goto done; 1947 css_get(&mem->css); 1948 } else { 1949 struct task_struct *p; 1950 1951 rcu_read_lock(); 1952 p = rcu_dereference(mm->owner); 1953 /* 1954 * Because we don't have task_lock(), "p" can exit. 1955 * In that case, "mem" can point to root or p can be NULL with 1956 * race with swapoff. Then, we have small risk of mis-accouning. 1957 * But such kind of mis-account by race always happens because 1958 * we don't have cgroup_mutex(). It's overkill and we allo that 1959 * small race, here. 1960 * (*) swapoff at el will charge against mm-struct not against 1961 * task-struct. So, mm->owner can be NULL. 1962 */ 1963 mem = mem_cgroup_from_task(p); 1964 if (!mem || mem_cgroup_is_root(mem)) { 1965 rcu_read_unlock(); 1966 goto done; 1967 } 1968 if (page_size == PAGE_SIZE && consume_stock(mem)) { 1969 /* 1970 * It seems dagerous to access memcg without css_get(). 1971 * But considering how consume_stok works, it's not 1972 * necessary. If consume_stock success, some charges 1973 * from this memcg are cached on this cpu. So, we 1974 * don't need to call css_get()/css_tryget() before 1975 * calling consume_stock(). 1976 */ 1977 rcu_read_unlock(); 1978 goto done; 1979 } 1980 /* after here, we may be blocked. we need to get refcnt */ 1981 if (!css_tryget(&mem->css)) { 1982 rcu_read_unlock(); 1983 goto again; 1984 } 1985 rcu_read_unlock(); 1986 } 1987 1988 do { 1989 bool oom_check; 1990 1991 /* If killed, bypass charge */ 1992 if (fatal_signal_pending(current)) { 1993 css_put(&mem->css); 1994 goto bypass; 1995 } 1996 1997 oom_check = false; 1998 if (oom && !nr_oom_retries) { 1999 oom_check = true; 2000 nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES; 2001 } 2002 2003 ret = __mem_cgroup_do_charge(mem, gfp_mask, csize, oom_check); 2004 2005 switch (ret) { 2006 case CHARGE_OK: 2007 break; 2008 case CHARGE_RETRY: /* not in OOM situation but retry */ 2009 csize = page_size; 2010 css_put(&mem->css); 2011 mem = NULL; 2012 goto again; 2013 case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */ 2014 css_put(&mem->css); 2015 goto nomem; 2016 case CHARGE_NOMEM: /* OOM routine works */ 2017 if (!oom) { 2018 css_put(&mem->css); 2019 goto nomem; 2020 } 2021 /* If oom, we never return -ENOMEM */ 2022 nr_oom_retries--; 2023 break; 2024 case CHARGE_OOM_DIE: /* Killed by OOM Killer */ 2025 css_put(&mem->css); 2026 goto bypass; 2027 } 2028 } while (ret != CHARGE_OK); 2029 2030 if (csize > page_size) 2031 refill_stock(mem, csize - page_size); 2032 css_put(&mem->css); 2033done: 2034 *memcg = mem; 2035 return 0; 2036nomem: 2037 *memcg = NULL; 2038 return -ENOMEM; 2039bypass: 2040 *memcg = NULL; 2041 return 0; 2042} 2043 2044/* 2045 * Somemtimes we have to undo a charge we got by try_charge(). 2046 * This function is for that and do uncharge, put css's refcnt. 2047 * gotten by try_charge(). 2048 */ 2049static void __mem_cgroup_cancel_charge(struct mem_cgroup *mem, 2050 unsigned long count) 2051{ 2052 if (!mem_cgroup_is_root(mem)) { 2053 res_counter_uncharge(&mem->res, PAGE_SIZE * count); 2054 if (do_swap_account) 2055 res_counter_uncharge(&mem->memsw, PAGE_SIZE * count); 2056 } 2057} 2058 2059static void mem_cgroup_cancel_charge(struct mem_cgroup *mem, 2060 int page_size) 2061{ 2062 __mem_cgroup_cancel_charge(mem, page_size >> PAGE_SHIFT); 2063} 2064 2065/* 2066 * A helper function to get mem_cgroup from ID. must be called under 2067 * rcu_read_lock(). The caller must check css_is_removed() or some if 2068 * it's concern. (dropping refcnt from swap can be called against removed 2069 * memcg.) 2070 */ 2071static struct mem_cgroup *mem_cgroup_lookup(unsigned short id) 2072{ 2073 struct cgroup_subsys_state *css; 2074 2075 /* ID 0 is unused ID */ 2076 if (!id) 2077 return NULL; 2078 css = css_lookup(&mem_cgroup_subsys, id); 2079 if (!css) 2080 return NULL; 2081 return container_of(css, struct mem_cgroup, css); 2082} 2083 2084struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page) 2085{ 2086 struct mem_cgroup *mem = NULL; 2087 struct page_cgroup *pc; 2088 unsigned short id; 2089 swp_entry_t ent; 2090 2091 VM_BUG_ON(!PageLocked(page)); 2092 2093 pc = lookup_page_cgroup(page); 2094 lock_page_cgroup(pc); 2095 if (PageCgroupUsed(pc)) { 2096 mem = pc->mem_cgroup; 2097 if (mem && !css_tryget(&mem->css)) 2098 mem = NULL; 2099 } else if (PageSwapCache(page)) { 2100 ent.val = page_private(page); 2101 id = lookup_swap_cgroup(ent); 2102 rcu_read_lock(); 2103 mem = mem_cgroup_lookup(id); 2104 if (mem && !css_tryget(&mem->css)) 2105 mem = NULL; 2106 rcu_read_unlock(); 2107 } 2108 unlock_page_cgroup(pc); 2109 return mem; 2110} 2111 2112static void __mem_cgroup_commit_charge(struct mem_cgroup *mem, 2113 struct page *page, 2114 struct page_cgroup *pc, 2115 enum charge_type ctype, 2116 int page_size) 2117{ 2118 int nr_pages = page_size >> PAGE_SHIFT; 2119 2120 lock_page_cgroup(pc); 2121 if (unlikely(PageCgroupUsed(pc))) { 2122 unlock_page_cgroup(pc); 2123 mem_cgroup_cancel_charge(mem, page_size); 2124 return; 2125 } 2126 /* 2127 * we don't need page_cgroup_lock about tail pages, becase they are not 2128 * accessed by any other context at this point. 2129 */ 2130 pc->mem_cgroup = mem; 2131 /* 2132 * We access a page_cgroup asynchronously without lock_page_cgroup(). 2133 * Especially when a page_cgroup is taken from a page, pc->mem_cgroup 2134 * is accessed after testing USED bit. To make pc->mem_cgroup visible 2135 * before USED bit, we need memory barrier here. 2136 * See mem_cgroup_add_lru_list(), etc. 2137 */ 2138 smp_wmb(); 2139 switch (ctype) { 2140 case MEM_CGROUP_CHARGE_TYPE_CACHE: 2141 case MEM_CGROUP_CHARGE_TYPE_SHMEM: 2142 SetPageCgroupCache(pc); 2143 SetPageCgroupUsed(pc); 2144 break; 2145 case MEM_CGROUP_CHARGE_TYPE_MAPPED: 2146 ClearPageCgroupCache(pc); 2147 SetPageCgroupUsed(pc); 2148 break; 2149 default: 2150 break; 2151 } 2152 2153 mem_cgroup_charge_statistics(mem, PageCgroupCache(pc), nr_pages); 2154 unlock_page_cgroup(pc); 2155 /* 2156 * "charge_statistics" updated event counter. Then, check it. 2157 * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree. 2158 * if they exceeds softlimit. 2159 */ 2160 memcg_check_events(mem, page); 2161} 2162 2163#ifdef CONFIG_TRANSPARENT_HUGEPAGE 2164 2165#define PCGF_NOCOPY_AT_SPLIT ((1 << PCG_LOCK) | (1 << PCG_MOVE_LOCK) |\ 2166 (1 << PCG_ACCT_LRU) | (1 << PCG_MIGRATION)) 2167/* 2168 * Because tail pages are not marked as "used", set it. We're under 2169 * zone->lru_lock, 'splitting on pmd' and compund_lock. 2170 */ 2171void mem_cgroup_split_huge_fixup(struct page *head, struct page *tail) 2172{ 2173 struct page_cgroup *head_pc = lookup_page_cgroup(head); 2174 struct page_cgroup *tail_pc = lookup_page_cgroup(tail); 2175 unsigned long flags; 2176 2177 if (mem_cgroup_disabled()) 2178 return; 2179 /* 2180 * We have no races with charge/uncharge but will have races with 2181 * page state accounting. 2182 */ 2183 move_lock_page_cgroup(head_pc, &flags); 2184 2185 tail_pc->mem_cgroup = head_pc->mem_cgroup; 2186 smp_wmb(); /* see __commit_charge() */ 2187 if (PageCgroupAcctLRU(head_pc)) { 2188 enum lru_list lru; 2189 struct mem_cgroup_per_zone *mz; 2190 2191 /* 2192 * LRU flags cannot be copied because we need to add tail 2193 *.page to LRU by generic call and our hook will be called. 2194 * We hold lru_lock, then, reduce counter directly. 2195 */ 2196 lru = page_lru(head); 2197 mz = page_cgroup_zoneinfo(head_pc->mem_cgroup, head); 2198 MEM_CGROUP_ZSTAT(mz, lru) -= 1; 2199 } 2200 tail_pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT; 2201 move_unlock_page_cgroup(head_pc, &flags); 2202} 2203#endif 2204 2205/** 2206 * mem_cgroup_move_account - move account of the page 2207 * @page: the page 2208 * @pc: page_cgroup of the page. 2209 * @from: mem_cgroup which the page is moved from. 2210 * @to: mem_cgroup which the page is moved to. @from != @to. 2211 * @uncharge: whether we should call uncharge and css_put against @from. 2212 * @charge_size: number of bytes to charge (regular or huge page) 2213 * 2214 * The caller must confirm following. 2215 * - page is not on LRU (isolate_page() is useful.) 2216 * - compound_lock is held when charge_size > PAGE_SIZE 2217 * 2218 * This function doesn't do "charge" nor css_get to new cgroup. It should be 2219 * done by a caller(__mem_cgroup_try_charge would be usefull). If @uncharge is 2220 * true, this function does "uncharge" from old cgroup, but it doesn't if 2221 * @uncharge is false, so a caller should do "uncharge". 2222 */ 2223static int mem_cgroup_move_account(struct page *page, struct page_cgroup *pc, 2224 struct mem_cgroup *from, struct mem_cgroup *to, 2225 bool uncharge, int charge_size) 2226{ 2227 int nr_pages = charge_size >> PAGE_SHIFT; 2228 unsigned long flags; 2229 int ret; 2230 2231 VM_BUG_ON(from == to); 2232 VM_BUG_ON(PageLRU(page)); 2233 /* 2234 * The page is isolated from LRU. So, collapse function 2235 * will not handle this page. But page splitting can happen. 2236 * Do this check under compound_page_lock(). The caller should 2237 * hold it. 2238 */ 2239 ret = -EBUSY; 2240 if (charge_size > PAGE_SIZE && !PageTransHuge(page)) 2241 goto out; 2242 2243 lock_page_cgroup(pc); 2244 2245 ret = -EINVAL; 2246 if (!PageCgroupUsed(pc) || pc->mem_cgroup != from) 2247 goto unlock; 2248 2249 move_lock_page_cgroup(pc, &flags); 2250 2251 if (PageCgroupFileMapped(pc)) { 2252 /* Update mapped_file data for mem_cgroup */ 2253 preempt_disable(); 2254 __this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]); 2255 __this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]); 2256 preempt_enable(); 2257 } 2258 mem_cgroup_charge_statistics(from, PageCgroupCache(pc), -nr_pages); 2259 if (uncharge) 2260 /* This is not "cancel", but cancel_charge does all we need. */ 2261 mem_cgroup_cancel_charge(from, charge_size); 2262 2263 /* caller should have done css_get */ 2264 pc->mem_cgroup = to; 2265 mem_cgroup_charge_statistics(to, PageCgroupCache(pc), nr_pages); 2266 /* 2267 * We charges against "to" which may not have any tasks. Then, "to" 2268 * can be under rmdir(). But in current implementation, caller of 2269 * this function is just force_empty() and move charge, so it's 2270 * garanteed that "to" is never removed. So, we don't check rmdir 2271 * status here. 2272 */ 2273 move_unlock_page_cgroup(pc, &flags); 2274 ret = 0; 2275unlock: 2276 unlock_page_cgroup(pc); 2277 /* 2278 * check events 2279 */ 2280 memcg_check_events(to, page); 2281 memcg_check_events(from, page); 2282out: 2283 return ret; 2284} 2285 2286/* 2287 * move charges to its parent. 2288 */ 2289 2290static int mem_cgroup_move_parent(struct page *page, 2291 struct page_cgroup *pc, 2292 struct mem_cgroup *child, 2293 gfp_t gfp_mask) 2294{ 2295 struct cgroup *cg = child->css.cgroup; 2296 struct cgroup *pcg = cg->parent; 2297 struct mem_cgroup *parent; 2298 int page_size = PAGE_SIZE; 2299 unsigned long flags; 2300 int ret; 2301 2302 /* Is ROOT ? */ 2303 if (!pcg) 2304 return -EINVAL; 2305 2306 ret = -EBUSY; 2307 if (!get_page_unless_zero(page)) 2308 goto out; 2309 if (isolate_lru_page(page)) 2310 goto put; 2311 2312 if (PageTransHuge(page)) 2313 page_size = HPAGE_SIZE; 2314 2315 parent = mem_cgroup_from_cont(pcg); 2316 ret = __mem_cgroup_try_charge(NULL, gfp_mask, 2317 &parent, false, page_size); 2318 if (ret || !parent) 2319 goto put_back; 2320 2321 if (page_size > PAGE_SIZE) 2322 flags = compound_lock_irqsave(page); 2323 2324 ret = mem_cgroup_move_account(page, pc, child, parent, true, page_size); 2325 if (ret) 2326 mem_cgroup_cancel_charge(parent, page_size); 2327 2328 if (page_size > PAGE_SIZE) 2329 compound_unlock_irqrestore(page, flags); 2330put_back: 2331 putback_lru_page(page); 2332put: 2333 put_page(page); 2334out: 2335 return ret; 2336} 2337 2338/* 2339 * Charge the memory controller for page usage. 2340 * Return 2341 * 0 if the charge was successful 2342 * < 0 if the cgroup is over its limit 2343 */ 2344static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm, 2345 gfp_t gfp_mask, enum charge_type ctype) 2346{ 2347 struct mem_cgroup *mem = NULL; 2348 int page_size = PAGE_SIZE; 2349 struct page_cgroup *pc; 2350 bool oom = true; 2351 int ret; 2352 2353 if (PageTransHuge(page)) { 2354 page_size <<= compound_order(page); 2355 VM_BUG_ON(!PageTransHuge(page)); 2356 /* 2357 * Never OOM-kill a process for a huge page. The 2358 * fault handler will fall back to regular pages. 2359 */ 2360 oom = false; 2361 } 2362 2363 pc = lookup_page_cgroup(page); 2364 BUG_ON(!pc); /* XXX: remove this and move pc lookup into commit */ 2365 2366 ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, oom, page_size); 2367 if (ret || !mem) 2368 return ret; 2369 2370 __mem_cgroup_commit_charge(mem, page, pc, ctype, page_size); 2371 return 0; 2372} 2373 2374int mem_cgroup_newpage_charge(struct page *page, 2375 struct mm_struct *mm, gfp_t gfp_mask) 2376{ 2377 if (mem_cgroup_disabled()) 2378 return 0; 2379 /* 2380 * If already mapped, we don't have to account. 2381 * If page cache, page->mapping has address_space. 2382 * But page->mapping may have out-of-use anon_vma pointer, 2383 * detecit it by PageAnon() check. newly-mapped-anon's page->mapping 2384 * is NULL. 2385 */ 2386 if (page_mapped(page) || (page->mapping && !PageAnon(page))) 2387 return 0; 2388 if (unlikely(!mm)) 2389 mm = &init_mm; 2390 return mem_cgroup_charge_common(page, mm, gfp_mask, 2391 MEM_CGROUP_CHARGE_TYPE_MAPPED); 2392} 2393 2394static void 2395__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr, 2396 enum charge_type ctype); 2397 2398int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm, 2399 gfp_t gfp_mask) 2400{ 2401 int ret; 2402 2403 if (mem_cgroup_disabled()) 2404 return 0; 2405 if (PageCompound(page)) 2406 return 0; 2407 /* 2408 * Corner case handling. This is called from add_to_page_cache() 2409 * in usual. But some FS (shmem) precharges this page before calling it 2410 * and call add_to_page_cache() with GFP_NOWAIT. 2411 * 2412 * For GFP_NOWAIT case, the page may be pre-charged before calling 2413 * add_to_page_cache(). (See shmem.c) check it here and avoid to call 2414 * charge twice. (It works but has to pay a bit larger cost.) 2415 * And when the page is SwapCache, it should take swap information 2416 * into account. This is under lock_page() now. 2417 */ 2418 if (!(gfp_mask & __GFP_WAIT)) { 2419 struct page_cgroup *pc; 2420 2421 pc = lookup_page_cgroup(page); 2422 if (!pc) 2423 return 0; 2424 lock_page_cgroup(pc); 2425 if (PageCgroupUsed(pc)) { 2426 unlock_page_cgroup(pc); 2427 return 0; 2428 } 2429 unlock_page_cgroup(pc); 2430 } 2431 2432 if (unlikely(!mm)) 2433 mm = &init_mm; 2434 2435 if (page_is_file_cache(page)) 2436 return mem_cgroup_charge_common(page, mm, gfp_mask, 2437 MEM_CGROUP_CHARGE_TYPE_CACHE); 2438 2439 /* shmem */ 2440 if (PageSwapCache(page)) { 2441 struct mem_cgroup *mem; 2442 2443 ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem); 2444 if (!ret) 2445 __mem_cgroup_commit_charge_swapin(page, mem, 2446 MEM_CGROUP_CHARGE_TYPE_SHMEM); 2447 } else 2448 ret = mem_cgroup_charge_common(page, mm, gfp_mask, 2449 MEM_CGROUP_CHARGE_TYPE_SHMEM); 2450 2451 return ret; 2452} 2453 2454/* 2455 * While swap-in, try_charge -> commit or cancel, the page is locked. 2456 * And when try_charge() successfully returns, one refcnt to memcg without 2457 * struct page_cgroup is acquired. This refcnt will be consumed by 2458 * "commit()" or removed by "cancel()" 2459 */ 2460int mem_cgroup_try_charge_swapin(struct mm_struct *mm, 2461 struct page *page, 2462 gfp_t mask, struct mem_cgroup **ptr) 2463{ 2464 struct mem_cgroup *mem; 2465 int ret; 2466 2467 *ptr = NULL; 2468 2469 if (mem_cgroup_disabled()) 2470 return 0; 2471 2472 if (!do_swap_account) 2473 goto charge_cur_mm; 2474 /* 2475 * A racing thread's fault, or swapoff, may have already updated 2476 * the pte, and even removed page from swap cache: in those cases 2477 * do_swap_page()'s pte_same() test will fail; but there's also a 2478 * KSM case which does need to charge the page. 2479 */ 2480 if (!PageSwapCache(page)) 2481 goto charge_cur_mm; 2482 mem = try_get_mem_cgroup_from_page(page); 2483 if (!mem) 2484 goto charge_cur_mm; 2485 *ptr = mem; 2486 ret = __mem_cgroup_try_charge(NULL, mask, ptr, true, PAGE_SIZE); 2487 css_put(&mem->css); 2488 return ret; 2489charge_cur_mm: 2490 if (unlikely(!mm)) 2491 mm = &init_mm; 2492 return __mem_cgroup_try_charge(mm, mask, ptr, true, PAGE_SIZE); 2493} 2494 2495static void 2496__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr, 2497 enum charge_type ctype) 2498{ 2499 struct page_cgroup *pc; 2500 2501 if (mem_cgroup_disabled()) 2502 return; 2503 if (!ptr) 2504 return; 2505 cgroup_exclude_rmdir(&ptr->css); 2506 pc = lookup_page_cgroup(page); 2507 mem_cgroup_lru_del_before_commit_swapcache(page); 2508 __mem_cgroup_commit_charge(ptr, page, pc, ctype, PAGE_SIZE); 2509 mem_cgroup_lru_add_after_commit_swapcache(page); 2510 /* 2511 * Now swap is on-memory. This means this page may be 2512 * counted both as mem and swap....double count. 2513 * Fix it by uncharging from memsw. Basically, this SwapCache is stable 2514 * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page() 2515 * may call delete_from_swap_cache() before reach here. 2516 */ 2517 if (do_swap_account && PageSwapCache(page)) { 2518 swp_entry_t ent = {.val = page_private(page)}; 2519 unsigned short id; 2520 struct mem_cgroup *memcg; 2521 2522 id = swap_cgroup_record(ent, 0); 2523 rcu_read_lock(); 2524 memcg = mem_cgroup_lookup(id); 2525 if (memcg) { 2526 /* 2527 * This recorded memcg can be obsolete one. So, avoid 2528 * calling css_tryget 2529 */ 2530 if (!mem_cgroup_is_root(memcg)) 2531 res_counter_uncharge(&memcg->memsw, PAGE_SIZE); 2532 mem_cgroup_swap_statistics(memcg, false); 2533 mem_cgroup_put(memcg); 2534 } 2535 rcu_read_unlock(); 2536 } 2537 /* 2538 * At swapin, we may charge account against cgroup which has no tasks. 2539 * So, rmdir()->pre_destroy() can be called while we do this charge. 2540 * In that case, we need to call pre_destroy() again. check it here. 2541 */ 2542 cgroup_release_and_wakeup_rmdir(&ptr->css); 2543} 2544 2545void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr) 2546{ 2547 __mem_cgroup_commit_charge_swapin(page, ptr, 2548 MEM_CGROUP_CHARGE_TYPE_MAPPED); 2549} 2550 2551void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem) 2552{ 2553 if (mem_cgroup_disabled()) 2554 return; 2555 if (!mem) 2556 return; 2557 mem_cgroup_cancel_charge(mem, PAGE_SIZE); 2558} 2559 2560static void 2561__do_uncharge(struct mem_cgroup *mem, const enum charge_type ctype, 2562 int page_size) 2563{ 2564 struct memcg_batch_info *batch = NULL; 2565 bool uncharge_memsw = true; 2566 /* If swapout, usage of swap doesn't decrease */ 2567 if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) 2568 uncharge_memsw = false; 2569 2570 batch = ¤t->memcg_batch; 2571 /* 2572 * In usual, we do css_get() when we remember memcg pointer. 2573 * But in this case, we keep res->usage until end of a series of 2574 * uncharges. Then, it's ok to ignore memcg's refcnt. 2575 */ 2576 if (!batch->memcg) 2577 batch->memcg = mem; 2578 /* 2579 * do_batch > 0 when unmapping pages or inode invalidate/truncate. 2580 * In those cases, all pages freed continously can be expected to be in 2581 * the same cgroup and we have chance to coalesce uncharges. 2582 * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE) 2583 * because we want to do uncharge as soon as possible. 2584 */ 2585 2586 if (!batch->do_batch || test_thread_flag(TIF_MEMDIE)) 2587 goto direct_uncharge; 2588 2589 if (page_size != PAGE_SIZE) 2590 goto direct_uncharge; 2591 2592 /* 2593 * In typical case, batch->memcg == mem. This means we can 2594 * merge a series of uncharges to an uncharge of res_counter. 2595 * If not, we uncharge res_counter ony by one. 2596 */ 2597 if (batch->memcg != mem) 2598 goto direct_uncharge; 2599 /* remember freed charge and uncharge it later */ 2600 batch->bytes += PAGE_SIZE; 2601 if (uncharge_memsw) 2602 batch->memsw_bytes += PAGE_SIZE; 2603 return; 2604direct_uncharge: 2605 res_counter_uncharge(&mem->res, page_size); 2606 if (uncharge_memsw) 2607 res_counter_uncharge(&mem->memsw, page_size); 2608 if (unlikely(batch->memcg != mem)) 2609 memcg_oom_recover(mem); 2610 return; 2611} 2612 2613/* 2614 * uncharge if !page_mapped(page) 2615 */ 2616static struct mem_cgroup * 2617__mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype) 2618{ 2619 int count; 2620 struct page_cgroup *pc; 2621 struct mem_cgroup *mem = NULL; 2622 int page_size = PAGE_SIZE; 2623 2624 if (mem_cgroup_disabled()) 2625 return NULL; 2626 2627 if (PageSwapCache(page)) 2628 return NULL; 2629 2630 if (PageTransHuge(page)) { 2631 page_size <<= compound_order(page); 2632 VM_BUG_ON(!PageTransHuge(page)); 2633 } 2634 2635 count = page_size >> PAGE_SHIFT; 2636 /* 2637 * Check if our page_cgroup is valid 2638 */ 2639 pc = lookup_page_cgroup(page); 2640 if (unlikely(!pc || !PageCgroupUsed(pc))) 2641 return NULL; 2642 2643 lock_page_cgroup(pc); 2644 2645 mem = pc->mem_cgroup; 2646 2647 if (!PageCgroupUsed(pc)) 2648 goto unlock_out; 2649 2650 switch (ctype) { 2651 case MEM_CGROUP_CHARGE_TYPE_MAPPED: 2652 case MEM_CGROUP_CHARGE_TYPE_DROP: 2653 /* See mem_cgroup_prepare_migration() */ 2654 if (page_mapped(page) || PageCgroupMigration(pc)) 2655 goto unlock_out; 2656 break; 2657 case MEM_CGROUP_CHARGE_TYPE_SWAPOUT: 2658 if (!PageAnon(page)) { /* Shared memory */ 2659 if (page->mapping && !page_is_file_cache(page)) 2660 goto unlock_out; 2661 } else if (page_mapped(page)) /* Anon */ 2662 goto unlock_out; 2663 break; 2664 default: 2665 break; 2666 } 2667 2668 mem_cgroup_charge_statistics(mem, PageCgroupCache(pc), -count); 2669 2670 ClearPageCgroupUsed(pc); 2671 /* 2672 * pc->mem_cgroup is not cleared here. It will be accessed when it's 2673 * freed from LRU. This is safe because uncharged page is expected not 2674 * to be reused (freed soon). Exception is SwapCache, it's handled by 2675 * special functions. 2676 */ 2677 2678 unlock_page_cgroup(pc); 2679 /* 2680 * even after unlock, we have mem->res.usage here and this memcg 2681 * will never be freed. 2682 */ 2683 memcg_check_events(mem, page); 2684 if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) { 2685 mem_cgroup_swap_statistics(mem, true); 2686 mem_cgroup_get(mem); 2687 } 2688 if (!mem_cgroup_is_root(mem)) 2689 __do_uncharge(mem, ctype, page_size); 2690 2691 return mem; 2692 2693unlock_out: 2694 unlock_page_cgroup(pc); 2695 return NULL; 2696} 2697 2698void mem_cgroup_uncharge_page(struct page *page) 2699{ 2700 /* early check. */ 2701 if (page_mapped(page)) 2702 return; 2703 if (page->mapping && !PageAnon(page)) 2704 return; 2705 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED); 2706} 2707 2708void mem_cgroup_uncharge_cache_page(struct page *page) 2709{ 2710 VM_BUG_ON(page_mapped(page)); 2711 VM_BUG_ON(page->mapping); 2712 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE); 2713} 2714 2715/* 2716 * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate. 2717 * In that cases, pages are freed continuously and we can expect pages 2718 * are in the same memcg. All these calls itself limits the number of 2719 * pages freed at once, then uncharge_start/end() is called properly. 2720 * This may be called prural(2) times in a context, 2721 */ 2722 2723void mem_cgroup_uncharge_start(void) 2724{ 2725 current->memcg_batch.do_batch++; 2726 /* We can do nest. */ 2727 if (current->memcg_batch.do_batch == 1) { 2728 current->memcg_batch.memcg = NULL; 2729 current->memcg_batch.bytes = 0; 2730 current->memcg_batch.memsw_bytes = 0; 2731 } 2732} 2733 2734void mem_cgroup_uncharge_end(void) 2735{ 2736 struct memcg_batch_info *batch = ¤t->memcg_batch; 2737 2738 if (!batch->do_batch) 2739 return; 2740 2741 batch->do_batch--; 2742 if (batch->do_batch) /* If stacked, do nothing. */ 2743 return; 2744 2745 if (!batch->memcg) 2746 return; 2747 /* 2748 * This "batch->memcg" is valid without any css_get/put etc... 2749 * bacause we hide charges behind us. 2750 */ 2751 if (batch->bytes) 2752 res_counter_uncharge(&batch->memcg->res, batch->bytes); 2753 if (batch->memsw_bytes) 2754 res_counter_uncharge(&batch->memcg->memsw, batch->memsw_bytes); 2755 memcg_oom_recover(batch->memcg); 2756 /* forget this pointer (for sanity check) */ 2757 batch->memcg = NULL; 2758} 2759 2760#ifdef CONFIG_SWAP 2761/* 2762 * called after __delete_from_swap_cache() and drop "page" account. 2763 * memcg information is recorded to swap_cgroup of "ent" 2764 */ 2765void 2766mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout) 2767{ 2768 struct mem_cgroup *memcg; 2769 int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT; 2770 2771 if (!swapout) /* this was a swap cache but the swap is unused ! */ 2772 ctype = MEM_CGROUP_CHARGE_TYPE_DROP; 2773 2774 memcg = __mem_cgroup_uncharge_common(page, ctype); 2775 2776 /* 2777 * record memcg information, if swapout && memcg != NULL, 2778 * mem_cgroup_get() was called in uncharge(). 2779 */ 2780 if (do_swap_account && swapout && memcg) 2781 swap_cgroup_record(ent, css_id(&memcg->css)); 2782} 2783#endif 2784 2785#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP 2786/* 2787 * called from swap_entry_free(). remove record in swap_cgroup and 2788 * uncharge "memsw" account. 2789 */ 2790void mem_cgroup_uncharge_swap(swp_entry_t ent) 2791{ 2792 struct mem_cgroup *memcg; 2793 unsigned short id; 2794 2795 if (!do_swap_account) 2796 return; 2797 2798 id = swap_cgroup_record(ent, 0); 2799 rcu_read_lock(); 2800 memcg = mem_cgroup_lookup(id); 2801 if (memcg) { 2802 /* 2803 * We uncharge this because swap is freed. 2804 * This memcg can be obsolete one. We avoid calling css_tryget 2805 */ 2806 if (!mem_cgroup_is_root(memcg)) 2807 res_counter_uncharge(&memcg->memsw, PAGE_SIZE); 2808 mem_cgroup_swap_statistics(memcg, false); 2809 mem_cgroup_put(memcg); 2810 } 2811 rcu_read_unlock(); 2812} 2813 2814/** 2815 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record. 2816 * @entry: swap entry to be moved 2817 * @from: mem_cgroup which the entry is moved from 2818 * @to: mem_cgroup which the entry is moved to 2819 * @need_fixup: whether we should fixup res_counters and refcounts. 2820 * 2821 * It succeeds only when the swap_cgroup's record for this entry is the same 2822 * as the mem_cgroup's id of @from. 2823 * 2824 * Returns 0 on success, -EINVAL on failure. 2825 * 2826 * The caller must have charged to @to, IOW, called res_counter_charge() about 2827 * both res and memsw, and called css_get(). 2828 */ 2829static int mem_cgroup_move_swap_account(swp_entry_t entry, 2830 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup) 2831{ 2832 unsigned short old_id, new_id; 2833 2834 old_id = css_id(&from->css); 2835 new_id = css_id(&to->css); 2836 2837 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) { 2838 mem_cgroup_swap_statistics(from, false); 2839 mem_cgroup_swap_statistics(to, true); 2840 /* 2841 * This function is only called from task migration context now. 2842 * It postpones res_counter and refcount handling till the end 2843 * of task migration(mem_cgroup_clear_mc()) for performance 2844 * improvement. But we cannot postpone mem_cgroup_get(to) 2845 * because if the process that has been moved to @to does 2846 * swap-in, the refcount of @to might be decreased to 0. 2847 */ 2848 mem_cgroup_get(to); 2849 if (need_fixup) { 2850 if (!mem_cgroup_is_root(from)) 2851 res_counter_uncharge(&from->memsw, PAGE_SIZE); 2852 mem_cgroup_put(from); 2853 /* 2854 * we charged both to->res and to->memsw, so we should 2855 * uncharge to->res. 2856 */ 2857 if (!mem_cgroup_is_root(to)) 2858 res_counter_uncharge(&to->res, PAGE_SIZE); 2859 } 2860 return 0; 2861 } 2862 return -EINVAL; 2863} 2864#else 2865static inline int mem_cgroup_move_swap_account(swp_entry_t entry, 2866 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup) 2867{ 2868 return -EINVAL; 2869} 2870#endif 2871 2872/* 2873 * Before starting migration, account PAGE_SIZE to mem_cgroup that the old 2874 * page belongs to. 2875 */ 2876int mem_cgroup_prepare_migration(struct page *page, 2877 struct page *newpage, struct mem_cgroup **ptr, gfp_t gfp_mask) 2878{ 2879 struct page_cgroup *pc; 2880 struct mem_cgroup *mem = NULL; 2881 enum charge_type ctype; 2882 int ret = 0; 2883 2884 *ptr = NULL; 2885 2886 VM_BUG_ON(PageTransHuge(page)); 2887 if (mem_cgroup_disabled()) 2888 return 0; 2889 2890 pc = lookup_page_cgroup(page); 2891 lock_page_cgroup(pc); 2892 if (PageCgroupUsed(pc)) { 2893 mem = pc->mem_cgroup; 2894 css_get(&mem->css); 2895 /* 2896 * At migrating an anonymous page, its mapcount goes down 2897 * to 0 and uncharge() will be called. But, even if it's fully 2898 * unmapped, migration may fail and this page has to be 2899 * charged again. We set MIGRATION flag here and delay uncharge 2900 * until end_migration() is called 2901 * 2902 * Corner Case Thinking 2903 * A) 2904 * When the old page was mapped as Anon and it's unmap-and-freed 2905 * while migration was ongoing. 2906 * If unmap finds the old page, uncharge() of it will be delayed 2907 * until end_migration(). If unmap finds a new page, it's 2908 * uncharged when it make mapcount to be 1->0. If unmap code 2909 * finds swap_migration_entry, the new page will not be mapped 2910 * and end_migration() will find it(mapcount==0). 2911 * 2912 * B) 2913 * When the old page was mapped but migraion fails, the kernel 2914 * remaps it. A charge for it is kept by MIGRATION flag even 2915 * if mapcount goes down to 0. We can do remap successfully 2916 * without charging it again. 2917 * 2918 * C) 2919 * The "old" page is under lock_page() until the end of 2920 * migration, so, the old page itself will not be swapped-out. 2921 * If the new page is swapped out before end_migraton, our 2922 * hook to usual swap-out path will catch the event. 2923 */ 2924 if (PageAnon(page)) 2925 SetPageCgroupMigration(pc); 2926 } 2927 unlock_page_cgroup(pc); 2928 /* 2929 * If the page is not charged at this point, 2930 * we return here. 2931 */ 2932 if (!mem) 2933 return 0; 2934 2935 *ptr = mem; 2936 ret = __mem_cgroup_try_charge(NULL, gfp_mask, ptr, false, PAGE_SIZE); 2937 css_put(&mem->css);/* drop extra refcnt */ 2938 if (ret || *ptr == NULL) { 2939 if (PageAnon(page)) { 2940 lock_page_cgroup(pc); 2941 ClearPageCgroupMigration(pc); 2942 unlock_page_cgroup(pc); 2943 /* 2944 * The old page may be fully unmapped while we kept it. 2945 */ 2946 mem_cgroup_uncharge_page(page); 2947 } 2948 return -ENOMEM; 2949 } 2950 /* 2951 * We charge new page before it's used/mapped. So, even if unlock_page() 2952 * is called before end_migration, we can catch all events on this new 2953 * page. In the case new page is migrated but not remapped, new page's 2954 * mapcount will be finally 0 and we call uncharge in end_migration(). 2955 */ 2956 pc = lookup_page_cgroup(newpage); 2957 if (PageAnon(page)) 2958 ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED; 2959 else if (page_is_file_cache(page)) 2960 ctype = MEM_CGROUP_CHARGE_TYPE_CACHE; 2961 else 2962 ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM; 2963 __mem_cgroup_commit_charge(mem, page, pc, ctype, PAGE_SIZE); 2964 return ret; 2965} 2966 2967/* remove redundant charge if migration failed*/ 2968void mem_cgroup_end_migration(struct mem_cgroup *mem, 2969 struct page *oldpage, struct page *newpage, bool migration_ok) 2970{ 2971 struct page *used, *unused; 2972 struct page_cgroup *pc; 2973 2974 if (!mem) 2975 return; 2976 /* blocks rmdir() */ 2977 cgroup_exclude_rmdir(&mem->css); 2978 if (!migration_ok) { 2979 used = oldpage; 2980 unused = newpage; 2981 } else { 2982 used = newpage; 2983 unused = oldpage; 2984 } 2985 /* 2986 * We disallowed uncharge of pages under migration because mapcount 2987 * of the page goes down to zero, temporarly. 2988 * Clear the flag and check the page should be charged. 2989 */ 2990 pc = lookup_page_cgroup(oldpage); 2991 lock_page_cgroup(pc); 2992 ClearPageCgroupMigration(pc); 2993 unlock_page_cgroup(pc); 2994 2995 __mem_cgroup_uncharge_common(unused, MEM_CGROUP_CHARGE_TYPE_FORCE); 2996 2997 /* 2998 * If a page is a file cache, radix-tree replacement is very atomic 2999 * and we can skip this check. When it was an Anon page, its mapcount 3000 * goes down to 0. But because we added MIGRATION flage, it's not 3001 * uncharged yet. There are several case but page->mapcount check 3002 * and USED bit check in mem_cgroup_uncharge_page() will do enough 3003 * check. (see prepare_charge() also) 3004 */ 3005 if (PageAnon(used)) 3006 mem_cgroup_uncharge_page(used); 3007 /* 3008 * At migration, we may charge account against cgroup which has no 3009 * tasks. 3010 * So, rmdir()->pre_destroy() can be called while we do this charge. 3011 * In that case, we need to call pre_destroy() again. check it here. 3012 */ 3013 cgroup_release_and_wakeup_rmdir(&mem->css); 3014} 3015 3016/* 3017 * A call to try to shrink memory usage on charge failure at shmem's swapin. 3018 * Calling hierarchical_reclaim is not enough because we should update 3019 * last_oom_jiffies to prevent pagefault_out_of_memory from invoking global OOM. 3020 * Moreover considering hierarchy, we should reclaim from the mem_over_limit, 3021 * not from the memcg which this page would be charged to. 3022 * try_charge_swapin does all of these works properly. 3023 */ 3024int mem_cgroup_shmem_charge_fallback(struct page *page, 3025 struct mm_struct *mm, 3026 gfp_t gfp_mask) 3027{ 3028 struct mem_cgroup *mem; 3029 int ret; 3030 3031 if (mem_cgroup_disabled()) 3032 return 0; 3033 3034 ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem); 3035 if (!ret) 3036 mem_cgroup_cancel_charge_swapin(mem); /* it does !mem check */ 3037 3038 return ret; 3039} 3040 3041#ifdef CONFIG_DEBUG_VM 3042static struct page_cgroup *lookup_page_cgroup_used(struct page *page) 3043{ 3044 struct page_cgroup *pc; 3045 3046 pc = lookup_page_cgroup(page); 3047 if (likely(pc) && PageCgroupUsed(pc)) 3048 return pc; 3049 return NULL; 3050} 3051 3052bool mem_cgroup_bad_page_check(struct page *page) 3053{ 3054 if (mem_cgroup_disabled()) 3055 return false; 3056 3057 return lookup_page_cgroup_used(page) != NULL; 3058} 3059 3060void mem_cgroup_print_bad_page(struct page *page) 3061{ 3062 struct page_cgroup *pc; 3063 3064 pc = lookup_page_cgroup_used(page); 3065 if (pc) { 3066 int ret = -1; 3067 char *path; 3068 3069 printk(KERN_ALERT "pc:%p pc->flags:%lx pc->mem_cgroup:%p", 3070 pc, pc->flags, pc->mem_cgroup); 3071 3072 path = kmalloc(PATH_MAX, GFP_KERNEL); 3073 if (path) { 3074 rcu_read_lock(); 3075 ret = cgroup_path(pc->mem_cgroup->css.cgroup, 3076 path, PATH_MAX); 3077 rcu_read_unlock(); 3078 } 3079 3080 printk(KERN_CONT "(%s)\n", 3081 (ret < 0) ? "cannot get the path" : path); 3082 kfree(path); 3083 } 3084} 3085#endif 3086 3087static DEFINE_MUTEX(set_limit_mutex); 3088 3089static int mem_cgroup_resize_limit(struct mem_cgroup *memcg, 3090 unsigned long long val) 3091{ 3092 int retry_count; 3093 u64 memswlimit, memlimit; 3094 int ret = 0; 3095 int children = mem_cgroup_count_children(memcg); 3096 u64 curusage, oldusage; 3097 int enlarge; 3098 3099 /* 3100 * For keeping hierarchical_reclaim simple, how long we should retry 3101 * is depends on callers. We set our retry-count to be function 3102 * of # of children which we should visit in this loop. 3103 */ 3104 retry_count = MEM_CGROUP_RECLAIM_RETRIES * children; 3105 3106 oldusage = res_counter_read_u64(&memcg->res, RES_USAGE); 3107 3108 enlarge = 0; 3109 while (retry_count) { 3110 if (signal_pending(current)) { 3111 ret = -EINTR; 3112 break; 3113 } 3114 /* 3115 * Rather than hide all in some function, I do this in 3116 * open coded manner. You see what this really does. 3117 * We have to guarantee mem->res.limit < mem->memsw.limit. 3118 */ 3119 mutex_lock(&set_limit_mutex); 3120 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 3121 if (memswlimit < val) { 3122 ret = -EINVAL; 3123 mutex_unlock(&set_limit_mutex); 3124 break; 3125 } 3126 3127 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT); 3128 if (memlimit < val) 3129 enlarge = 1; 3130 3131 ret = res_counter_set_limit(&memcg->res, val); 3132 if (!ret) { 3133 if (memswlimit == val) 3134 memcg->memsw_is_minimum = true; 3135 else 3136 memcg->memsw_is_minimum = false; 3137 } 3138 mutex_unlock(&set_limit_mutex); 3139 3140 if (!ret) 3141 break; 3142 3143 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL, 3144 MEM_CGROUP_RECLAIM_SHRINK); 3145 curusage = res_counter_read_u64(&memcg->res, RES_USAGE); 3146 /* Usage is reduced ? */ 3147 if (curusage >= oldusage) 3148 retry_count--; 3149 else 3150 oldusage = curusage; 3151 } 3152 if (!ret && enlarge) 3153 memcg_oom_recover(memcg); 3154 3155 return ret; 3156} 3157 3158static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg, 3159 unsigned long long val) 3160{ 3161 int retry_count; 3162 u64 memlimit, memswlimit, oldusage, curusage; 3163 int children = mem_cgroup_count_children(memcg); 3164 int ret = -EBUSY; 3165 int enlarge = 0; 3166 3167 /* see mem_cgroup_resize_res_limit */ 3168 retry_count = children * MEM_CGROUP_RECLAIM_RETRIES; 3169 oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE); 3170 while (retry_count) { 3171 if (signal_pending(current)) { 3172 ret = -EINTR; 3173 break; 3174 } 3175 /* 3176 * Rather than hide all in some function, I do this in 3177 * open coded manner. You see what this really does. 3178 * We have to guarantee mem->res.limit < mem->memsw.limit. 3179 */ 3180 mutex_lock(&set_limit_mutex); 3181 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT); 3182 if (memlimit > val) { 3183 ret = -EINVAL; 3184 mutex_unlock(&set_limit_mutex); 3185 break; 3186 } 3187 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 3188 if (memswlimit < val) 3189 enlarge = 1; 3190 ret = res_counter_set_limit(&memcg->memsw, val); 3191 if (!ret) { 3192 if (memlimit == val) 3193 memcg->memsw_is_minimum = true; 3194 else 3195 memcg->memsw_is_minimum = false; 3196 } 3197 mutex_unlock(&set_limit_mutex); 3198 3199 if (!ret) 3200 break; 3201 3202 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL, 3203 MEM_CGROUP_RECLAIM_NOSWAP | 3204 MEM_CGROUP_RECLAIM_SHRINK); 3205 curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE); 3206 /* Usage is reduced ? */ 3207 if (curusage >= oldusage) 3208 retry_count--; 3209 else 3210 oldusage = curusage; 3211 } 3212 if (!ret && enlarge) 3213 memcg_oom_recover(memcg); 3214 return ret; 3215} 3216 3217unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order, 3218 gfp_t gfp_mask) 3219{ 3220 unsigned long nr_reclaimed = 0; 3221 struct mem_cgroup_per_zone *mz, *next_mz = NULL; 3222 unsigned long reclaimed; 3223 int loop = 0; 3224 struct mem_cgroup_tree_per_zone *mctz; 3225 unsigned long long excess; 3226 3227 if (order > 0) 3228 return 0; 3229 3230 mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone)); 3231 /* 3232 * This loop can run a while, specially if mem_cgroup's continuously 3233 * keep exceeding their soft limit and putting the system under 3234 * pressure 3235 */ 3236 do { 3237 if (next_mz) 3238 mz = next_mz; 3239 else 3240 mz = mem_cgroup_largest_soft_limit_node(mctz); 3241 if (!mz) 3242 break; 3243 3244 reclaimed = mem_cgroup_hierarchical_reclaim(mz->mem, zone, 3245 gfp_mask, 3246 MEM_CGROUP_RECLAIM_SOFT); 3247 nr_reclaimed += reclaimed; 3248 spin_lock(&mctz->lock); 3249 3250 /* 3251 * If we failed to reclaim anything from this memory cgroup 3252 * it is time to move on to the next cgroup 3253 */ 3254 next_mz = NULL; 3255 if (!reclaimed) { 3256 do { 3257 /* 3258 * Loop until we find yet another one. 3259 * 3260 * By the time we get the soft_limit lock 3261 * again, someone might have aded the 3262 * group back on the RB tree. Iterate to 3263 * make sure we get a different mem. 3264 * mem_cgroup_largest_soft_limit_node returns 3265 * NULL if no other cgroup is present on 3266 * the tree 3267 */ 3268 next_mz = 3269 __mem_cgroup_largest_soft_limit_node(mctz); 3270 if (next_mz == mz) { 3271 css_put(&next_mz->mem->css); 3272 next_mz = NULL; 3273 } else /* next_mz == NULL or other memcg */ 3274 break; 3275 } while (1); 3276 } 3277 __mem_cgroup_remove_exceeded(mz->mem, mz, mctz); 3278 excess = res_counter_soft_limit_excess(&mz->mem->res); 3279 /* 3280 * One school of thought says that we should not add 3281 * back the node to the tree if reclaim returns 0. 3282 * But our reclaim could return 0, simply because due 3283 * to priority we are exposing a smaller subset of 3284 * memory to reclaim from. Consider this as a longer 3285 * term TODO. 3286 */ 3287 /* If excess == 0, no tree ops */ 3288 __mem_cgroup_insert_exceeded(mz->mem, mz, mctz, excess); 3289 spin_unlock(&mctz->lock); 3290 css_put(&mz->mem->css); 3291 loop++; 3292 /* 3293 * Could not reclaim anything and there are no more 3294 * mem cgroups to try or we seem to be looping without 3295 * reclaiming anything. 3296 */ 3297 if (!nr_reclaimed && 3298 (next_mz == NULL || 3299 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS)) 3300 break; 3301 } while (!nr_reclaimed); 3302 if (next_mz) 3303 css_put(&next_mz->mem->css); 3304 return nr_reclaimed; 3305} 3306 3307/* 3308 * This routine traverse page_cgroup in given list and drop them all. 3309 * *And* this routine doesn't reclaim page itself, just removes page_cgroup. 3310 */ 3311static int mem_cgroup_force_empty_list(struct mem_cgroup *mem, 3312 int node, int zid, enum lru_list lru) 3313{ 3314 struct zone *zone; 3315 struct mem_cgroup_per_zone *mz; 3316 struct page_cgroup *pc, *busy; 3317 unsigned long flags, loop; 3318 struct list_head *list; 3319 int ret = 0; 3320 3321 zone = &NODE_DATA(node)->node_zones[zid]; 3322 mz = mem_cgroup_zoneinfo(mem, node, zid); 3323 list = &mz->lists[lru]; 3324 3325 loop = MEM_CGROUP_ZSTAT(mz, lru); 3326 /* give some margin against EBUSY etc...*/ 3327 loop += 256; 3328 busy = NULL; 3329 while (loop--) { 3330 struct page *page; 3331 3332 ret = 0; 3333 spin_lock_irqsave(&zone->lru_lock, flags); 3334 if (list_empty(list)) { 3335 spin_unlock_irqrestore(&zone->lru_lock, flags); 3336 break; 3337 } 3338 pc = list_entry(list->prev, struct page_cgroup, lru); 3339 if (busy == pc) { 3340 list_move(&pc->lru, list); 3341 busy = NULL; 3342 spin_unlock_irqrestore(&zone->lru_lock, flags); 3343 continue; 3344 } 3345 spin_unlock_irqrestore(&zone->lru_lock, flags); 3346 3347 page = pc->page; 3348 3349 ret = mem_cgroup_move_parent(page, pc, mem, GFP_KERNEL); 3350 if (ret == -ENOMEM) 3351 break; 3352 3353 if (ret == -EBUSY || ret == -EINVAL) { 3354 /* found lock contention or "pc" is obsolete. */ 3355 busy = pc; 3356 cond_resched(); 3357 } else 3358 busy = NULL; 3359 } 3360 3361 if (!ret && !list_empty(list)) 3362 return -EBUSY; 3363 return ret; 3364} 3365 3366/* 3367 * make mem_cgroup's charge to be 0 if there is no task. 3368 * This enables deleting this mem_cgroup. 3369 */ 3370static int mem_cgroup_force_empty(struct mem_cgroup *mem, bool free_all) 3371{ 3372 int ret; 3373 int node, zid, shrink; 3374 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES; 3375 struct cgroup *cgrp = mem->css.cgroup; 3376 3377 css_get(&mem->css); 3378 3379 shrink = 0; 3380 /* should free all ? */ 3381 if (free_all) 3382 goto try_to_free; 3383move_account: 3384 do { 3385 ret = -EBUSY; 3386 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children)) 3387 goto out; 3388 ret = -EINTR; 3389 if (signal_pending(current)) 3390 goto out; 3391 /* This is for making all *used* pages to be on LRU. */ 3392 lru_add_drain_all(); 3393 drain_all_stock_sync(); 3394 ret = 0; 3395 mem_cgroup_start_move(mem); 3396 for_each_node_state(node, N_HIGH_MEMORY) { 3397 for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) { 3398 enum lru_list l; 3399 for_each_lru(l) { 3400 ret = mem_cgroup_force_empty_list(mem, 3401 node, zid, l); 3402 if (ret) 3403 break; 3404 } 3405 } 3406 if (ret) 3407 break; 3408 } 3409 mem_cgroup_end_move(mem); 3410 memcg_oom_recover(mem); 3411 /* it seems parent cgroup doesn't have enough mem */ 3412 if (ret == -ENOMEM) 3413 goto try_to_free; 3414 cond_resched(); 3415 /* "ret" should also be checked to ensure all lists are empty. */ 3416 } while (mem->res.usage > 0 || ret); 3417out: 3418 css_put(&mem->css); 3419 return ret; 3420 3421try_to_free: 3422 /* returns EBUSY if there is a task or if we come here twice. */ 3423 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children) || shrink) { 3424 ret = -EBUSY; 3425 goto out; 3426 } 3427 /* we call try-to-free pages for make this cgroup empty */ 3428 lru_add_drain_all(); 3429 /* try to free all pages in this cgroup */ 3430 shrink = 1; 3431 while (nr_retries && mem->res.usage > 0) { 3432 int progress; 3433 3434 if (signal_pending(current)) { 3435 ret = -EINTR; 3436 goto out; 3437 } 3438 progress = try_to_free_mem_cgroup_pages(mem, GFP_KERNEL, 3439 false, get_swappiness(mem)); 3440 if (!progress) { 3441 nr_retries--; 3442 /* maybe some writeback is necessary */ 3443 congestion_wait(BLK_RW_ASYNC, HZ/10); 3444 } 3445 3446 } 3447 lru_add_drain(); 3448 /* try move_account...there may be some *locked* pages. */ 3449 goto move_account; 3450} 3451 3452int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event) 3453{ 3454 return mem_cgroup_force_empty(mem_cgroup_from_cont(cont), true); 3455} 3456 3457 3458static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft) 3459{ 3460 return mem_cgroup_from_cont(cont)->use_hierarchy; 3461} 3462 3463static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft, 3464 u64 val) 3465{ 3466 int retval = 0; 3467 struct mem_cgroup *mem = mem_cgroup_from_cont(cont); 3468 struct cgroup *parent = cont->parent; 3469 struct mem_cgroup *parent_mem = NULL; 3470 3471 if (parent) 3472 parent_mem = mem_cgroup_from_cont(parent); 3473 3474 cgroup_lock(); 3475 /* 3476 * If parent's use_hierarchy is set, we can't make any modifications 3477 * in the child subtrees. If it is unset, then the change can 3478 * occur, provided the current cgroup has no children. 3479 * 3480 * For the root cgroup, parent_mem is NULL, we allow value to be 3481 * set if there are no children. 3482 */ 3483 if ((!parent_mem || !parent_mem->use_hierarchy) && 3484 (val == 1 || val == 0)) { 3485 if (list_empty(&cont->children)) 3486 mem->use_hierarchy = val; 3487 else 3488 retval = -EBUSY; 3489 } else 3490 retval = -EINVAL; 3491 cgroup_unlock(); 3492 3493 return retval; 3494} 3495 3496 3497static u64 mem_cgroup_get_recursive_idx_stat(struct mem_cgroup *mem, 3498 enum mem_cgroup_stat_index idx) 3499{ 3500 struct mem_cgroup *iter; 3501 s64 val = 0; 3502 3503 /* each per cpu's value can be minus.Then, use s64 */ 3504 for_each_mem_cgroup_tree(iter, mem) 3505 val += mem_cgroup_read_stat(iter, idx); 3506 3507 if (val < 0) /* race ? */ 3508 val = 0; 3509 return val; 3510} 3511 3512static inline u64 mem_cgroup_usage(struct mem_cgroup *mem, bool swap) 3513{ 3514 u64 val; 3515 3516 if (!mem_cgroup_is_root(mem)) { 3517 if (!swap) 3518 return res_counter_read_u64(&mem->res, RES_USAGE); 3519 else 3520 return res_counter_read_u64(&mem->memsw, RES_USAGE); 3521 } 3522 3523 val = mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_CACHE); 3524 val += mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_RSS); 3525 3526 if (swap) 3527 val += mem_cgroup_get_recursive_idx_stat(mem, 3528 MEM_CGROUP_STAT_SWAPOUT); 3529 3530 return val << PAGE_SHIFT; 3531} 3532 3533static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft) 3534{ 3535 struct mem_cgroup *mem = mem_cgroup_from_cont(cont); 3536 u64 val; 3537 int type, name; 3538 3539 type = MEMFILE_TYPE(cft->private); 3540 name = MEMFILE_ATTR(cft->private); 3541 switch (type) { 3542 case _MEM: 3543 if (name == RES_USAGE) 3544 val = mem_cgroup_usage(mem, false); 3545 else 3546 val = res_counter_read_u64(&mem->res, name); 3547 break; 3548 case _MEMSWAP: 3549 if (name == RES_USAGE) 3550 val = mem_cgroup_usage(mem, true); 3551 else 3552 val = res_counter_read_u64(&mem->memsw, name); 3553 break; 3554 default: 3555 BUG(); 3556 break; 3557 } 3558 return val; 3559} 3560/* 3561 * The user of this function is... 3562 * RES_LIMIT. 3563 */ 3564static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft, 3565 const char *buffer) 3566{ 3567 struct mem_cgroup *memcg = mem_cgroup_from_cont(cont); 3568 int type, name; 3569 unsigned long long val; 3570 int ret; 3571 3572 type = MEMFILE_TYPE(cft->private); 3573 name = MEMFILE_ATTR(cft->private); 3574 switch (name) { 3575 case RES_LIMIT: 3576 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */ 3577 ret = -EINVAL; 3578 break; 3579 } 3580 /* This function does all necessary parse...reuse it */ 3581 ret = res_counter_memparse_write_strategy(buffer, &val); 3582 if (ret) 3583 break; 3584 if (type == _MEM) 3585 ret = mem_cgroup_resize_limit(memcg, val); 3586 else 3587 ret = mem_cgroup_resize_memsw_limit(memcg, val); 3588 break; 3589 case RES_SOFT_LIMIT: 3590 ret = res_counter_memparse_write_strategy(buffer, &val); 3591 if (ret) 3592 break; 3593 /* 3594 * For memsw, soft limits are hard to implement in terms 3595 * of semantics, for now, we support soft limits for 3596 * control without swap 3597 */ 3598 if (type == _MEM) 3599 ret = res_counter_set_soft_limit(&memcg->res, val); 3600 else 3601 ret = -EINVAL; 3602 break; 3603 default: 3604 ret = -EINVAL; /* should be BUG() ? */ 3605 break; 3606 } 3607 return ret; 3608} 3609 3610static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg, 3611 unsigned long long *mem_limit, unsigned long long *memsw_limit) 3612{ 3613 struct cgroup *cgroup; 3614 unsigned long long min_limit, min_memsw_limit, tmp; 3615 3616 min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT); 3617 min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 3618 cgroup = memcg->css.cgroup; 3619 if (!memcg->use_hierarchy) 3620 goto out; 3621 3622 while (cgroup->parent) { 3623 cgroup = cgroup->parent; 3624 memcg = mem_cgroup_from_cont(cgroup); 3625 if (!memcg->use_hierarchy) 3626 break; 3627 tmp = res_counter_read_u64(&memcg->res, RES_LIMIT); 3628 min_limit = min(min_limit, tmp); 3629 tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 3630 min_memsw_limit = min(min_memsw_limit, tmp); 3631 } 3632out: 3633 *mem_limit = min_limit; 3634 *memsw_limit = min_memsw_limit; 3635 return; 3636} 3637 3638static int mem_cgroup_reset(struct cgroup *cont, unsigned int event) 3639{ 3640 struct mem_cgroup *mem; 3641 int type, name; 3642 3643 mem = mem_cgroup_from_cont(cont); 3644 type = MEMFILE_TYPE(event); 3645 name = MEMFILE_ATTR(event); 3646 switch (name) { 3647 case RES_MAX_USAGE: 3648 if (type == _MEM) 3649 res_counter_reset_max(&mem->res); 3650 else 3651 res_counter_reset_max(&mem->memsw); 3652 break; 3653 case RES_FAILCNT: 3654 if (type == _MEM) 3655 res_counter_reset_failcnt(&mem->res); 3656 else 3657 res_counter_reset_failcnt(&mem->memsw); 3658 break; 3659 } 3660 3661 return 0; 3662} 3663 3664static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp, 3665 struct cftype *cft) 3666{ 3667 return mem_cgroup_from_cont(cgrp)->move_charge_at_immigrate; 3668} 3669 3670#ifdef CONFIG_MMU 3671static int mem_cgroup_move_charge_write(struct cgroup *cgrp, 3672 struct cftype *cft, u64 val) 3673{ 3674 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp); 3675 3676 if (val >= (1 << NR_MOVE_TYPE)) 3677 return -EINVAL; 3678 /* 3679 * We check this value several times in both in can_attach() and 3680 * attach(), so we need cgroup lock to prevent this value from being 3681 * inconsistent. 3682 */ 3683 cgroup_lock(); 3684 mem->move_charge_at_immigrate = val; 3685 cgroup_unlock(); 3686 3687 return 0; 3688} 3689#else 3690static int mem_cgroup_move_charge_write(struct cgroup *cgrp, 3691 struct cftype *cft, u64 val) 3692{ 3693 return -ENOSYS; 3694} 3695#endif 3696 3697 3698/* For read statistics */ 3699enum { 3700 MCS_CACHE, 3701 MCS_RSS, 3702 MCS_FILE_MAPPED, 3703 MCS_PGPGIN, 3704 MCS_PGPGOUT, 3705 MCS_SWAP, 3706 MCS_INACTIVE_ANON, 3707 MCS_ACTIVE_ANON, 3708 MCS_INACTIVE_FILE, 3709 MCS_ACTIVE_FILE, 3710 MCS_UNEVICTABLE, 3711 NR_MCS_STAT, 3712}; 3713 3714struct mcs_total_stat { 3715 s64 stat[NR_MCS_STAT]; 3716}; 3717 3718struct { 3719 char *local_name; 3720 char *total_name; 3721} memcg_stat_strings[NR_MCS_STAT] = { 3722 {"cache", "total_cache"}, 3723 {"rss", "total_rss"}, 3724 {"mapped_file", "total_mapped_file"}, 3725 {"pgpgin", "total_pgpgin"}, 3726 {"pgpgout", "total_pgpgout"}, 3727 {"swap", "total_swap"}, 3728 {"inactive_anon", "total_inactive_anon"}, 3729 {"active_anon", "total_active_anon"}, 3730 {"inactive_file", "total_inactive_file"}, 3731 {"active_file", "total_active_file"}, 3732 {"unevictable", "total_unevictable"} 3733}; 3734 3735 3736static void 3737mem_cgroup_get_local_stat(struct mem_cgroup *mem, struct mcs_total_stat *s) 3738{ 3739 s64 val; 3740 3741 /* per cpu stat */ 3742 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE); 3743 s->stat[MCS_CACHE] += val * PAGE_SIZE; 3744 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS); 3745 s->stat[MCS_RSS] += val * PAGE_SIZE; 3746 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_FILE_MAPPED); 3747 s->stat[MCS_FILE_MAPPED] += val * PAGE_SIZE; 3748 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGIN_COUNT); 3749 s->stat[MCS_PGPGIN] += val; 3750 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGOUT_COUNT); 3751 s->stat[MCS_PGPGOUT] += val; 3752 if (do_swap_account) { 3753 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_SWAPOUT); 3754 s->stat[MCS_SWAP] += val * PAGE_SIZE; 3755 } 3756 3757 /* per zone stat */ 3758 val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_ANON); 3759 s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE; 3760 val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_ANON); 3761 s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE; 3762 val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_FILE); 3763 s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE; 3764 val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_FILE); 3765 s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE; 3766 val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE); 3767 s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE; 3768} 3769 3770static void 3771mem_cgroup_get_total_stat(struct mem_cgroup *mem, struct mcs_total_stat *s) 3772{ 3773 struct mem_cgroup *iter; 3774 3775 for_each_mem_cgroup_tree(iter, mem) 3776 mem_cgroup_get_local_stat(iter, s); 3777} 3778 3779static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft, 3780 struct cgroup_map_cb *cb) 3781{ 3782 struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont); 3783 struct mcs_total_stat mystat; 3784 int i; 3785 3786 memset(&mystat, 0, sizeof(mystat)); 3787 mem_cgroup_get_local_stat(mem_cont, &mystat); 3788 3789 for (i = 0; i < NR_MCS_STAT; i++) { 3790 if (i == MCS_SWAP && !do_swap_account) 3791 continue; 3792 cb->fill(cb, memcg_stat_strings[i].local_name, mystat.stat[i]); 3793 } 3794 3795 /* Hierarchical information */ 3796 { 3797 unsigned long long limit, memsw_limit; 3798 memcg_get_hierarchical_limit(mem_cont, &limit, &memsw_limit); 3799 cb->fill(cb, "hierarchical_memory_limit", limit); 3800 if (do_swap_account) 3801 cb->fill(cb, "hierarchical_memsw_limit", memsw_limit); 3802 } 3803 3804 memset(&mystat, 0, sizeof(mystat)); 3805 mem_cgroup_get_total_stat(mem_cont, &mystat); 3806 for (i = 0; i < NR_MCS_STAT; i++) { 3807 if (i == MCS_SWAP && !do_swap_account) 3808 continue; 3809 cb->fill(cb, memcg_stat_strings[i].total_name, mystat.stat[i]); 3810 } 3811 3812#ifdef CONFIG_DEBUG_VM 3813 cb->fill(cb, "inactive_ratio", calc_inactive_ratio(mem_cont, NULL)); 3814 3815 { 3816 int nid, zid; 3817 struct mem_cgroup_per_zone *mz; 3818 unsigned long recent_rotated[2] = {0, 0}; 3819 unsigned long recent_scanned[2] = {0, 0}; 3820 3821 for_each_online_node(nid) 3822 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 3823 mz = mem_cgroup_zoneinfo(mem_cont, nid, zid); 3824 3825 recent_rotated[0] += 3826 mz->reclaim_stat.recent_rotated[0]; 3827 recent_rotated[1] += 3828 mz->reclaim_stat.recent_rotated[1]; 3829 recent_scanned[0] += 3830 mz->reclaim_stat.recent_scanned[0]; 3831 recent_scanned[1] += 3832 mz->reclaim_stat.recent_scanned[1]; 3833 } 3834 cb->fill(cb, "recent_rotated_anon", recent_rotated[0]); 3835 cb->fill(cb, "recent_rotated_file", recent_rotated[1]); 3836 cb->fill(cb, "recent_scanned_anon", recent_scanned[0]); 3837 cb->fill(cb, "recent_scanned_file", recent_scanned[1]); 3838 } 3839#endif 3840 3841 return 0; 3842} 3843 3844static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft) 3845{ 3846 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); 3847 3848 return get_swappiness(memcg); 3849} 3850 3851static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft, 3852 u64 val) 3853{ 3854 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); 3855 struct mem_cgroup *parent; 3856 3857 if (val > 100) 3858 return -EINVAL; 3859 3860 if (cgrp->parent == NULL) 3861 return -EINVAL; 3862 3863 parent = mem_cgroup_from_cont(cgrp->parent); 3864 3865 cgroup_lock(); 3866 3867 /* If under hierarchy, only empty-root can set this value */ 3868 if ((parent->use_hierarchy) || 3869 (memcg->use_hierarchy && !list_empty(&cgrp->children))) { 3870 cgroup_unlock(); 3871 return -EINVAL; 3872 } 3873 3874 spin_lock(&memcg->reclaim_param_lock); 3875 memcg->swappiness = val; 3876 spin_unlock(&memcg->reclaim_param_lock); 3877 3878 cgroup_unlock(); 3879 3880 return 0; 3881} 3882 3883static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap) 3884{ 3885 struct mem_cgroup_threshold_ary *t; 3886 u64 usage; 3887 int i; 3888 3889 rcu_read_lock(); 3890 if (!swap) 3891 t = rcu_dereference(memcg->thresholds.primary); 3892 else 3893 t = rcu_dereference(memcg->memsw_thresholds.primary); 3894 3895 if (!t) 3896 goto unlock; 3897 3898 usage = mem_cgroup_usage(memcg, swap); 3899 3900 /* 3901 * current_threshold points to threshold just below usage. 3902 * If it's not true, a threshold was crossed after last 3903 * call of __mem_cgroup_threshold(). 3904 */ 3905 i = t->current_threshold; 3906 3907 /* 3908 * Iterate backward over array of thresholds starting from 3909 * current_threshold and check if a threshold is crossed. 3910 * If none of thresholds below usage is crossed, we read 3911 * only one element of the array here. 3912 */ 3913 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--) 3914 eventfd_signal(t->entries[i].eventfd, 1); 3915 3916 /* i = current_threshold + 1 */ 3917 i++; 3918 3919 /* 3920 * Iterate forward over array of thresholds starting from 3921 * current_threshold+1 and check if a threshold is crossed. 3922 * If none of thresholds above usage is crossed, we read 3923 * only one element of the array here. 3924 */ 3925 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++) 3926 eventfd_signal(t->entries[i].eventfd, 1); 3927 3928 /* Update current_threshold */ 3929 t->current_threshold = i - 1; 3930unlock: 3931 rcu_read_unlock(); 3932} 3933 3934static void mem_cgroup_threshold(struct mem_cgroup *memcg) 3935{ 3936 while (memcg) { 3937 __mem_cgroup_threshold(memcg, false); 3938 if (do_swap_account) 3939 __mem_cgroup_threshold(memcg, true); 3940 3941 memcg = parent_mem_cgroup(memcg); 3942 } 3943} 3944 3945static int compare_thresholds(const void *a, const void *b) 3946{ 3947 const struct mem_cgroup_threshold *_a = a; 3948 const struct mem_cgroup_threshold *_b = b; 3949 3950 return _a->threshold - _b->threshold; 3951} 3952 3953static int mem_cgroup_oom_notify_cb(struct mem_cgroup *mem) 3954{ 3955 struct mem_cgroup_eventfd_list *ev; 3956 3957 list_for_each_entry(ev, &mem->oom_notify, list) 3958 eventfd_signal(ev->eventfd, 1); 3959 return 0; 3960} 3961 3962static void mem_cgroup_oom_notify(struct mem_cgroup *mem) 3963{ 3964 struct mem_cgroup *iter; 3965 3966 for_each_mem_cgroup_tree(iter, mem) 3967 mem_cgroup_oom_notify_cb(iter); 3968} 3969 3970static int mem_cgroup_usage_register_event(struct cgroup *cgrp, 3971 struct cftype *cft, struct eventfd_ctx *eventfd, const char *args) 3972{ 3973 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); 3974 struct mem_cgroup_thresholds *thresholds; 3975 struct mem_cgroup_threshold_ary *new; 3976 int type = MEMFILE_TYPE(cft->private); 3977 u64 threshold, usage; 3978 int i, size, ret; 3979 3980 ret = res_counter_memparse_write_strategy(args, &threshold); 3981 if (ret) 3982 return ret; 3983 3984 mutex_lock(&memcg->thresholds_lock); 3985 3986 if (type == _MEM) 3987 thresholds = &memcg->thresholds; 3988 else if (type == _MEMSWAP) 3989 thresholds = &memcg->memsw_thresholds; 3990 else 3991 BUG(); 3992 3993 usage = mem_cgroup_usage(memcg, type == _MEMSWAP); 3994 3995 /* Check if a threshold crossed before adding a new one */ 3996 if (thresholds->primary) 3997 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 3998 3999 size = thresholds->primary ? thresholds->primary->size + 1 : 1; 4000 4001 /* Allocate memory for new array of thresholds */ 4002 new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold), 4003 GFP_KERNEL); 4004 if (!new) { 4005 ret = -ENOMEM; 4006 goto unlock; 4007 } 4008 new->size = size; 4009 4010 /* Copy thresholds (if any) to new array */ 4011 if (thresholds->primary) { 4012 memcpy(new->entries, thresholds->primary->entries, (size - 1) * 4013 sizeof(struct mem_cgroup_threshold)); 4014 } 4015 4016 /* Add new threshold */ 4017 new->entries[size - 1].eventfd = eventfd; 4018 new->entries[size - 1].threshold = threshold; 4019 4020 /* Sort thresholds. Registering of new threshold isn't time-critical */ 4021 sort(new->entries, size, sizeof(struct mem_cgroup_threshold), 4022 compare_thresholds, NULL); 4023 4024 /* Find current threshold */ 4025 new->current_threshold = -1; 4026 for (i = 0; i < size; i++) { 4027 if (new->entries[i].threshold < usage) { 4028 /* 4029 * new->current_threshold will not be used until 4030 * rcu_assign_pointer(), so it's safe to increment 4031 * it here. 4032 */ 4033 ++new->current_threshold; 4034 } 4035 } 4036 4037 /* Free old spare buffer and save old primary buffer as spare */ 4038 kfree(thresholds->spare); 4039 thresholds->spare = thresholds->primary; 4040 4041 rcu_assign_pointer(thresholds->primary, new); 4042 4043 /* To be sure that nobody uses thresholds */ 4044 synchronize_rcu(); 4045 4046unlock: 4047 mutex_unlock(&memcg->thresholds_lock); 4048 4049 return ret; 4050} 4051 4052static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp, 4053 struct cftype *cft, struct eventfd_ctx *eventfd) 4054{ 4055 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); 4056 struct mem_cgroup_thresholds *thresholds; 4057 struct mem_cgroup_threshold_ary *new; 4058 int type = MEMFILE_TYPE(cft->private); 4059 u64 usage; 4060 int i, j, size; 4061 4062 mutex_lock(&memcg->thresholds_lock); 4063 if (type == _MEM) 4064 thresholds = &memcg->thresholds; 4065 else if (type == _MEMSWAP) 4066 thresholds = &memcg->memsw_thresholds; 4067 else 4068 BUG(); 4069 4070 /* 4071 * Something went wrong if we trying to unregister a threshold 4072 * if we don't have thresholds 4073 */ 4074 BUG_ON(!thresholds); 4075 4076 usage = mem_cgroup_usage(memcg, type == _MEMSWAP); 4077 4078 /* Check if a threshold crossed before removing */ 4079 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 4080 4081 /* Calculate new number of threshold */ 4082 size = 0; 4083 for (i = 0; i < thresholds->primary->size; i++) { 4084 if (thresholds->primary->entries[i].eventfd != eventfd) 4085 size++; 4086 } 4087 4088 new = thresholds->spare; 4089 4090 /* Set thresholds array to NULL if we don't have thresholds */ 4091 if (!size) { 4092 kfree(new); 4093 new = NULL; 4094 goto swap_buffers; 4095 } 4096 4097 new->size = size; 4098 4099 /* Copy thresholds and find current threshold */ 4100 new->current_threshold = -1; 4101 for (i = 0, j = 0; i < thresholds->primary->size; i++) { 4102 if (thresholds->primary->entries[i].eventfd == eventfd) 4103 continue; 4104 4105 new->entries[j] = thresholds->primary->entries[i]; 4106 if (new->entries[j].threshold < usage) { 4107 /* 4108 * new->current_threshold will not be used 4109 * until rcu_assign_pointer(), so it's safe to increment 4110 * it here. 4111 */ 4112 ++new->current_threshold; 4113 } 4114 j++; 4115 } 4116 4117swap_buffers: 4118 /* Swap primary and spare array */ 4119 thresholds->spare = thresholds->primary; 4120 rcu_assign_pointer(thresholds->primary, new); 4121 4122 /* To be sure that nobody uses thresholds */ 4123 synchronize_rcu(); 4124 4125 mutex_unlock(&memcg->thresholds_lock); 4126} 4127 4128static int mem_cgroup_oom_register_event(struct cgroup *cgrp, 4129 struct cftype *cft, struct eventfd_ctx *eventfd, const char *args) 4130{ 4131 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); 4132 struct mem_cgroup_eventfd_list *event; 4133 int type = MEMFILE_TYPE(cft->private); 4134 4135 BUG_ON(type != _OOM_TYPE); 4136 event = kmalloc(sizeof(*event), GFP_KERNEL); 4137 if (!event) 4138 return -ENOMEM; 4139 4140 mutex_lock(&memcg_oom_mutex); 4141 4142 event->eventfd = eventfd; 4143 list_add(&event->list, &memcg->oom_notify); 4144 4145 /* already in OOM ? */ 4146 if (atomic_read(&memcg->oom_lock)) 4147 eventfd_signal(eventfd, 1); 4148 mutex_unlock(&memcg_oom_mutex); 4149 4150 return 0; 4151} 4152 4153static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp, 4154 struct cftype *cft, struct eventfd_ctx *eventfd) 4155{ 4156 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp); 4157 struct mem_cgroup_eventfd_list *ev, *tmp; 4158 int type = MEMFILE_TYPE(cft->private); 4159 4160 BUG_ON(type != _OOM_TYPE); 4161 4162 mutex_lock(&memcg_oom_mutex); 4163 4164 list_for_each_entry_safe(ev, tmp, &mem->oom_notify, list) { 4165 if (ev->eventfd == eventfd) { 4166 list_del(&ev->list); 4167 kfree(ev); 4168 } 4169 } 4170 4171 mutex_unlock(&memcg_oom_mutex); 4172} 4173 4174static int mem_cgroup_oom_control_read(struct cgroup *cgrp, 4175 struct cftype *cft, struct cgroup_map_cb *cb) 4176{ 4177 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp); 4178 4179 cb->fill(cb, "oom_kill_disable", mem->oom_kill_disable); 4180 4181 if (atomic_read(&mem->oom_lock)) 4182 cb->fill(cb, "under_oom", 1); 4183 else 4184 cb->fill(cb, "under_oom", 0); 4185 return 0; 4186} 4187 4188static int mem_cgroup_oom_control_write(struct cgroup *cgrp, 4189 struct cftype *cft, u64 val) 4190{ 4191 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp); 4192 struct mem_cgroup *parent; 4193 4194 /* cannot set to root cgroup and only 0 and 1 are allowed */ 4195 if (!cgrp->parent || !((val == 0) || (val == 1))) 4196 return -EINVAL; 4197 4198 parent = mem_cgroup_from_cont(cgrp->parent); 4199 4200 cgroup_lock(); 4201 /* oom-kill-disable is a flag for subhierarchy. */ 4202 if ((parent->use_hierarchy) || 4203 (mem->use_hierarchy && !list_empty(&cgrp->children))) { 4204 cgroup_unlock(); 4205 return -EINVAL; 4206 } 4207 mem->oom_kill_disable = val; 4208 if (!val) 4209 memcg_oom_recover(mem); 4210 cgroup_unlock(); 4211 return 0; 4212} 4213 4214static struct cftype mem_cgroup_files[] = { 4215 { 4216 .name = "usage_in_bytes", 4217 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE), 4218 .read_u64 = mem_cgroup_read, 4219 .register_event = mem_cgroup_usage_register_event, 4220 .unregister_event = mem_cgroup_usage_unregister_event, 4221 }, 4222 { 4223 .name = "max_usage_in_bytes", 4224 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE), 4225 .trigger = mem_cgroup_reset, 4226 .read_u64 = mem_cgroup_read, 4227 }, 4228 { 4229 .name = "limit_in_bytes", 4230 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT), 4231 .write_string = mem_cgroup_write, 4232 .read_u64 = mem_cgroup_read, 4233 }, 4234 { 4235 .name = "soft_limit_in_bytes", 4236 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT), 4237 .write_string = mem_cgroup_write, 4238 .read_u64 = mem_cgroup_read, 4239 }, 4240 { 4241 .name = "failcnt", 4242 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT), 4243 .trigger = mem_cgroup_reset, 4244 .read_u64 = mem_cgroup_read, 4245 }, 4246 { 4247 .name = "stat", 4248 .read_map = mem_control_stat_show, 4249 }, 4250 { 4251 .name = "force_empty", 4252 .trigger = mem_cgroup_force_empty_write, 4253 }, 4254 { 4255 .name = "use_hierarchy", 4256 .write_u64 = mem_cgroup_hierarchy_write, 4257 .read_u64 = mem_cgroup_hierarchy_read, 4258 }, 4259 { 4260 .name = "swappiness", 4261 .read_u64 = mem_cgroup_swappiness_read, 4262 .write_u64 = mem_cgroup_swappiness_write, 4263 }, 4264 { 4265 .name = "move_charge_at_immigrate", 4266 .read_u64 = mem_cgroup_move_charge_read, 4267 .write_u64 = mem_cgroup_move_charge_write, 4268 }, 4269 { 4270 .name = "oom_control", 4271 .read_map = mem_cgroup_oom_control_read, 4272 .write_u64 = mem_cgroup_oom_control_write, 4273 .register_event = mem_cgroup_oom_register_event, 4274 .unregister_event = mem_cgroup_oom_unregister_event, 4275 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL), 4276 }, 4277}; 4278 4279#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP 4280static struct cftype memsw_cgroup_files[] = { 4281 { 4282 .name = "memsw.usage_in_bytes", 4283 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE), 4284 .read_u64 = mem_cgroup_read, 4285 .register_event = mem_cgroup_usage_register_event, 4286 .unregister_event = mem_cgroup_usage_unregister_event, 4287 }, 4288 { 4289 .name = "memsw.max_usage_in_bytes", 4290 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE), 4291 .trigger = mem_cgroup_reset, 4292 .read_u64 = mem_cgroup_read, 4293 }, 4294 { 4295 .name = "memsw.limit_in_bytes", 4296 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT), 4297 .write_string = mem_cgroup_write, 4298 .read_u64 = mem_cgroup_read, 4299 }, 4300 { 4301 .name = "memsw.failcnt", 4302 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT), 4303 .trigger = mem_cgroup_reset, 4304 .read_u64 = mem_cgroup_read, 4305 }, 4306}; 4307 4308static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss) 4309{ 4310 if (!do_swap_account) 4311 return 0; 4312 return cgroup_add_files(cont, ss, memsw_cgroup_files, 4313 ARRAY_SIZE(memsw_cgroup_files)); 4314}; 4315#else 4316static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss) 4317{ 4318 return 0; 4319} 4320#endif 4321 4322static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node) 4323{ 4324 struct mem_cgroup_per_node *pn; 4325 struct mem_cgroup_per_zone *mz; 4326 enum lru_list l; 4327 int zone, tmp = node; 4328 /* 4329 * This routine is called against possible nodes. 4330 * But it's BUG to call kmalloc() against offline node. 4331 * 4332 * TODO: this routine can waste much memory for nodes which will 4333 * never be onlined. It's better to use memory hotplug callback 4334 * function. 4335 */ 4336 if (!node_state(node, N_NORMAL_MEMORY)) 4337 tmp = -1; 4338 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp); 4339 if (!pn) 4340 return 1; 4341 4342 mem->info.nodeinfo[node] = pn; 4343 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 4344 mz = &pn->zoneinfo[zone]; 4345 for_each_lru(l) 4346 INIT_LIST_HEAD(&mz->lists[l]); 4347 mz->usage_in_excess = 0; 4348 mz->on_tree = false; 4349 mz->mem = mem; 4350 } 4351 return 0; 4352} 4353 4354static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node) 4355{ 4356 kfree(mem->info.nodeinfo[node]); 4357} 4358 4359static struct mem_cgroup *mem_cgroup_alloc(void) 4360{ 4361 struct mem_cgroup *mem; 4362 int size = sizeof(struct mem_cgroup); 4363 4364 /* Can be very big if MAX_NUMNODES is very big */ 4365 if (size < PAGE_SIZE) 4366 mem = kzalloc(size, GFP_KERNEL); 4367 else 4368 mem = vzalloc(size); 4369 4370 if (!mem) 4371 return NULL; 4372 4373 mem->stat = alloc_percpu(struct mem_cgroup_stat_cpu); 4374 if (!mem->stat) 4375 goto out_free; 4376 spin_lock_init(&mem->pcp_counter_lock); 4377 return mem; 4378 4379out_free: 4380 if (size < PAGE_SIZE) 4381 kfree(mem); 4382 else 4383 vfree(mem); 4384 return NULL; 4385} 4386 4387/* 4388 * At destroying mem_cgroup, references from swap_cgroup can remain. 4389 * (scanning all at force_empty is too costly...) 4390 * 4391 * Instead of clearing all references at force_empty, we remember 4392 * the number of reference from swap_cgroup and free mem_cgroup when 4393 * it goes down to 0. 4394 * 4395 * Removal of cgroup itself succeeds regardless of refs from swap. 4396 */ 4397 4398static void __mem_cgroup_free(struct mem_cgroup *mem) 4399{ 4400 int node; 4401 4402 mem_cgroup_remove_from_trees(mem); 4403 free_css_id(&mem_cgroup_subsys, &mem->css); 4404 4405 for_each_node_state(node, N_POSSIBLE) 4406 free_mem_cgroup_per_zone_info(mem, node); 4407 4408 free_percpu(mem->stat); 4409 if (sizeof(struct mem_cgroup) < PAGE_SIZE) 4410 kfree(mem); 4411 else 4412 vfree(mem); 4413} 4414 4415static void mem_cgroup_get(struct mem_cgroup *mem) 4416{ 4417 atomic_inc(&mem->refcnt); 4418} 4419 4420static void __mem_cgroup_put(struct mem_cgroup *mem, int count) 4421{ 4422 if (atomic_sub_and_test(count, &mem->refcnt)) { 4423 struct mem_cgroup *parent = parent_mem_cgroup(mem); 4424 __mem_cgroup_free(mem); 4425 if (parent) 4426 mem_cgroup_put(parent); 4427 } 4428} 4429 4430static void mem_cgroup_put(struct mem_cgroup *mem) 4431{ 4432 __mem_cgroup_put(mem, 1); 4433} 4434 4435/* 4436 * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled. 4437 */ 4438static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem) 4439{ 4440 if (!mem->res.parent) 4441 return NULL; 4442 return mem_cgroup_from_res_counter(mem->res.parent, res); 4443} 4444 4445#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP 4446static void __init enable_swap_cgroup(void) 4447{ 4448 if (!mem_cgroup_disabled() && really_do_swap_account) 4449 do_swap_account = 1; 4450} 4451#else 4452static void __init enable_swap_cgroup(void) 4453{ 4454} 4455#endif 4456 4457static int mem_cgroup_soft_limit_tree_init(void) 4458{ 4459 struct mem_cgroup_tree_per_node *rtpn; 4460 struct mem_cgroup_tree_per_zone *rtpz; 4461 int tmp, node, zone; 4462 4463 for_each_node_state(node, N_POSSIBLE) { 4464 tmp = node; 4465 if (!node_state(node, N_NORMAL_MEMORY)) 4466 tmp = -1; 4467 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp); 4468 if (!rtpn) 4469 return 1; 4470 4471 soft_limit_tree.rb_tree_per_node[node] = rtpn; 4472 4473 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 4474 rtpz = &rtpn->rb_tree_per_zone[zone]; 4475 rtpz->rb_root = RB_ROOT; 4476 spin_lock_init(&rtpz->lock); 4477 } 4478 } 4479 return 0; 4480} 4481 4482static struct cgroup_subsys_state * __ref 4483mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont) 4484{ 4485 struct mem_cgroup *mem, *parent; 4486 long error = -ENOMEM; 4487 int node; 4488 4489 mem = mem_cgroup_alloc(); 4490 if (!mem) 4491 return ERR_PTR(error); 4492 4493 for_each_node_state(node, N_POSSIBLE) 4494 if (alloc_mem_cgroup_per_zone_info(mem, node)) 4495 goto free_out; 4496 4497 /* root ? */ 4498 if (cont->parent == NULL) { 4499 int cpu; 4500 enable_swap_cgroup(); 4501 parent = NULL; 4502 root_mem_cgroup = mem; 4503 if (mem_cgroup_soft_limit_tree_init()) 4504 goto free_out; 4505 for_each_possible_cpu(cpu) { 4506 struct memcg_stock_pcp *stock = 4507 &per_cpu(memcg_stock, cpu); 4508 INIT_WORK(&stock->work, drain_local_stock); 4509 } 4510 hotcpu_notifier(memcg_cpu_hotplug_callback, 0); 4511 } else { 4512 parent = mem_cgroup_from_cont(cont->parent); 4513 mem->use_hierarchy = parent->use_hierarchy; 4514 mem->oom_kill_disable = parent->oom_kill_disable; 4515 } 4516 4517 if (parent && parent->use_hierarchy) { 4518 res_counter_init(&mem->res, &parent->res); 4519 res_counter_init(&mem->memsw, &parent->memsw); 4520 /* 4521 * We increment refcnt of the parent to ensure that we can 4522 * safely access it on res_counter_charge/uncharge. 4523 * This refcnt will be decremented when freeing this 4524 * mem_cgroup(see mem_cgroup_put). 4525 */ 4526 mem_cgroup_get(parent); 4527 } else { 4528 res_counter_init(&mem->res, NULL); 4529 res_counter_init(&mem->memsw, NULL); 4530 } 4531 mem->last_scanned_child = 0; 4532 spin_lock_init(&mem->reclaim_param_lock); 4533 INIT_LIST_HEAD(&mem->oom_notify); 4534 4535 if (parent) 4536 mem->swappiness = get_swappiness(parent); 4537 atomic_set(&mem->refcnt, 1); 4538 mem->move_charge_at_immigrate = 0; 4539 mutex_init(&mem->thresholds_lock); 4540 return &mem->css; 4541free_out: 4542 __mem_cgroup_free(mem); 4543 root_mem_cgroup = NULL; 4544 return ERR_PTR(error); 4545} 4546 4547static int mem_cgroup_pre_destroy(struct cgroup_subsys *ss, 4548 struct cgroup *cont) 4549{ 4550 struct mem_cgroup *mem = mem_cgroup_from_cont(cont); 4551 4552 return mem_cgroup_force_empty(mem, false); 4553} 4554 4555static void mem_cgroup_destroy(struct cgroup_subsys *ss, 4556 struct cgroup *cont) 4557{ 4558 struct mem_cgroup *mem = mem_cgroup_from_cont(cont); 4559 4560 mem_cgroup_put(mem); 4561} 4562 4563static int mem_cgroup_populate(struct cgroup_subsys *ss, 4564 struct cgroup *cont) 4565{ 4566 int ret; 4567 4568 ret = cgroup_add_files(cont, ss, mem_cgroup_files, 4569 ARRAY_SIZE(mem_cgroup_files)); 4570 4571 if (!ret) 4572 ret = register_memsw_files(cont, ss); 4573 return ret; 4574} 4575 4576#ifdef CONFIG_MMU 4577/* Handlers for move charge at task migration. */ 4578#define PRECHARGE_COUNT_AT_ONCE 256 4579static int mem_cgroup_do_precharge(unsigned long count) 4580{ 4581 int ret = 0; 4582 int batch_count = PRECHARGE_COUNT_AT_ONCE; 4583 struct mem_cgroup *mem = mc.to; 4584 4585 if (mem_cgroup_is_root(mem)) { 4586 mc.precharge += count; 4587 /* we don't need css_get for root */ 4588 return ret; 4589 } 4590 /* try to charge at once */ 4591 if (count > 1) { 4592 struct res_counter *dummy; 4593 /* 4594 * "mem" cannot be under rmdir() because we've already checked 4595 * by cgroup_lock_live_cgroup() that it is not removed and we 4596 * are still under the same cgroup_mutex. So we can postpone 4597 * css_get(). 4598 */ 4599 if (res_counter_charge(&mem->res, PAGE_SIZE * count, &dummy)) 4600 goto one_by_one; 4601 if (do_swap_account && res_counter_charge(&mem->memsw, 4602 PAGE_SIZE * count, &dummy)) { 4603 res_counter_uncharge(&mem->res, PAGE_SIZE * count); 4604 goto one_by_one; 4605 } 4606 mc.precharge += count; 4607 return ret; 4608 } 4609one_by_one: 4610 /* fall back to one by one charge */ 4611 while (count--) { 4612 if (signal_pending(current)) { 4613 ret = -EINTR; 4614 break; 4615 } 4616 if (!batch_count--) { 4617 batch_count = PRECHARGE_COUNT_AT_ONCE; 4618 cond_resched(); 4619 } 4620 ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem, false, 4621 PAGE_SIZE); 4622 if (ret || !mem) 4623 /* mem_cgroup_clear_mc() will do uncharge later */ 4624 return -ENOMEM; 4625 mc.precharge++; 4626 } 4627 return ret; 4628} 4629 4630/** 4631 * is_target_pte_for_mc - check a pte whether it is valid for move charge 4632 * @vma: the vma the pte to be checked belongs 4633 * @addr: the address corresponding to the pte to be checked 4634 * @ptent: the pte to be checked 4635 * @target: the pointer the target page or swap ent will be stored(can be NULL) 4636 * 4637 * Returns 4638 * 0(MC_TARGET_NONE): if the pte is not a target for move charge. 4639 * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for 4640 * move charge. if @target is not NULL, the page is stored in target->page 4641 * with extra refcnt got(Callers should handle it). 4642 * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a 4643 * target for charge migration. if @target is not NULL, the entry is stored 4644 * in target->ent. 4645 * 4646 * Called with pte lock held. 4647 */ 4648union mc_target { 4649 struct page *page; 4650 swp_entry_t ent; 4651}; 4652 4653enum mc_target_type { 4654 MC_TARGET_NONE, /* not used */ 4655 MC_TARGET_PAGE, 4656 MC_TARGET_SWAP, 4657}; 4658 4659static struct page *mc_handle_present_pte(struct vm_area_struct *vma, 4660 unsigned long addr, pte_t ptent) 4661{ 4662 struct page *page = vm_normal_page(vma, addr, ptent); 4663 4664 if (!page || !page_mapped(page)) 4665 return NULL; 4666 if (PageAnon(page)) { 4667 /* we don't move shared anon */ 4668 if (!move_anon() || page_mapcount(page) > 2) 4669 return NULL; 4670 } else if (!move_file()) 4671 /* we ignore mapcount for file pages */ 4672 return NULL; 4673 if (!get_page_unless_zero(page)) 4674 return NULL; 4675 4676 return page; 4677} 4678 4679static struct page *mc_handle_swap_pte(struct vm_area_struct *vma, 4680 unsigned long addr, pte_t ptent, swp_entry_t *entry) 4681{ 4682 int usage_count; 4683 struct page *page = NULL; 4684 swp_entry_t ent = pte_to_swp_entry(ptent); 4685 4686 if (!move_anon() || non_swap_entry(ent)) 4687 return NULL; 4688 usage_count = mem_cgroup_count_swap_user(ent, &page); 4689 if (usage_count > 1) { /* we don't move shared anon */ 4690 if (page) 4691 put_page(page); 4692 return NULL; 4693 } 4694 if (do_swap_account) 4695 entry->val = ent.val; 4696 4697 return page; 4698} 4699 4700static struct page *mc_handle_file_pte(struct vm_area_struct *vma, 4701 unsigned long addr, pte_t ptent, swp_entry_t *entry) 4702{ 4703 struct page *page = NULL; 4704 struct inode *inode; 4705 struct address_space *mapping; 4706 pgoff_t pgoff; 4707 4708 if (!vma->vm_file) /* anonymous vma */ 4709 return NULL; 4710 if (!move_file()) 4711 return NULL; 4712 4713 inode = vma->vm_file->f_path.dentry->d_inode; 4714 mapping = vma->vm_file->f_mapping; 4715 if (pte_none(ptent)) 4716 pgoff = linear_page_index(vma, addr); 4717 else /* pte_file(ptent) is true */ 4718 pgoff = pte_to_pgoff(ptent); 4719 4720 /* page is moved even if it's not RSS of this task(page-faulted). */ 4721 if (!mapping_cap_swap_backed(mapping)) { /* normal file */ 4722 page = find_get_page(mapping, pgoff); 4723 } else { /* shmem/tmpfs file. we should take account of swap too. */ 4724 swp_entry_t ent; 4725 mem_cgroup_get_shmem_target(inode, pgoff, &page, &ent); 4726 if (do_swap_account) 4727 entry->val = ent.val; 4728 } 4729 4730 return page; 4731} 4732 4733static int is_target_pte_for_mc(struct vm_area_struct *vma, 4734 unsigned long addr, pte_t ptent, union mc_target *target) 4735{ 4736 struct page *page = NULL; 4737 struct page_cgroup *pc; 4738 int ret = 0; 4739 swp_entry_t ent = { .val = 0 }; 4740 4741 if (pte_present(ptent)) 4742 page = mc_handle_present_pte(vma, addr, ptent); 4743 else if (is_swap_pte(ptent)) 4744 page = mc_handle_swap_pte(vma, addr, ptent, &ent); 4745 else if (pte_none(ptent) || pte_file(ptent)) 4746 page = mc_handle_file_pte(vma, addr, ptent, &ent); 4747 4748 if (!page && !ent.val) 4749 return 0; 4750 if (page) { 4751 pc = lookup_page_cgroup(page); 4752 /* 4753 * Do only loose check w/o page_cgroup lock. 4754 * mem_cgroup_move_account() checks the pc is valid or not under 4755 * the lock. 4756 */ 4757 if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) { 4758 ret = MC_TARGET_PAGE; 4759 if (target) 4760 target->page = page; 4761 } 4762 if (!ret || !target) 4763 put_page(page); 4764 } 4765 /* There is a swap entry and a page doesn't exist or isn't charged */ 4766 if (ent.val && !ret && 4767 css_id(&mc.from->css) == lookup_swap_cgroup(ent)) { 4768 ret = MC_TARGET_SWAP; 4769 if (target) 4770 target->ent = ent; 4771 } 4772 return ret; 4773} 4774 4775static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd, 4776 unsigned long addr, unsigned long end, 4777 struct mm_walk *walk) 4778{ 4779 struct vm_area_struct *vma = walk->private; 4780 pte_t *pte; 4781 spinlock_t *ptl; 4782 4783 split_huge_page_pmd(walk->mm, pmd); 4784 4785 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 4786 for (; addr != end; pte++, addr += PAGE_SIZE) 4787 if (is_target_pte_for_mc(vma, addr, *pte, NULL)) 4788 mc.precharge++; /* increment precharge temporarily */ 4789 pte_unmap_unlock(pte - 1, ptl); 4790 cond_resched(); 4791 4792 return 0; 4793} 4794 4795static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm) 4796{ 4797 unsigned long precharge; 4798 struct vm_area_struct *vma; 4799 4800 down_read(&mm->mmap_sem); 4801 for (vma = mm->mmap; vma; vma = vma->vm_next) { 4802 struct mm_walk mem_cgroup_count_precharge_walk = { 4803 .pmd_entry = mem_cgroup_count_precharge_pte_range, 4804 .mm = mm, 4805 .private = vma, 4806 }; 4807 if (is_vm_hugetlb_page(vma)) 4808 continue; 4809 walk_page_range(vma->vm_start, vma->vm_end, 4810 &mem_cgroup_count_precharge_walk); 4811 } 4812 up_read(&mm->mmap_sem); 4813 4814 precharge = mc.precharge; 4815 mc.precharge = 0; 4816 4817 return precharge; 4818} 4819 4820static int mem_cgroup_precharge_mc(struct mm_struct *mm) 4821{ 4822 unsigned long precharge = mem_cgroup_count_precharge(mm); 4823 4824 VM_BUG_ON(mc.moving_task); 4825 mc.moving_task = current; 4826 return mem_cgroup_do_precharge(precharge); 4827} 4828 4829/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */ 4830static void __mem_cgroup_clear_mc(void) 4831{ 4832 struct mem_cgroup *from = mc.from; 4833 struct mem_cgroup *to = mc.to; 4834 4835 /* we must uncharge all the leftover precharges from mc.to */ 4836 if (mc.precharge) { 4837 __mem_cgroup_cancel_charge(mc.to, mc.precharge); 4838 mc.precharge = 0; 4839 } 4840 /* 4841 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so 4842 * we must uncharge here. 4843 */ 4844 if (mc.moved_charge) { 4845 __mem_cgroup_cancel_charge(mc.from, mc.moved_charge); 4846 mc.moved_charge = 0; 4847 } 4848 /* we must fixup refcnts and charges */ 4849 if (mc.moved_swap) { 4850 /* uncharge swap account from the old cgroup */ 4851 if (!mem_cgroup_is_root(mc.from)) 4852 res_counter_uncharge(&mc.from->memsw, 4853 PAGE_SIZE * mc.moved_swap); 4854 __mem_cgroup_put(mc.from, mc.moved_swap); 4855 4856 if (!mem_cgroup_is_root(mc.to)) { 4857 /* 4858 * we charged both to->res and to->memsw, so we should 4859 * uncharge to->res. 4860 */ 4861 res_counter_uncharge(&mc.to->res, 4862 PAGE_SIZE * mc.moved_swap); 4863 } 4864 /* we've already done mem_cgroup_get(mc.to) */ 4865 mc.moved_swap = 0; 4866 } 4867 memcg_oom_recover(from); 4868 memcg_oom_recover(to); 4869 wake_up_all(&mc.waitq); 4870} 4871 4872static void mem_cgroup_clear_mc(void) 4873{ 4874 struct mem_cgroup *from = mc.from; 4875 4876 /* 4877 * we must clear moving_task before waking up waiters at the end of 4878 * task migration. 4879 */ 4880 mc.moving_task = NULL; 4881 __mem_cgroup_clear_mc(); 4882 spin_lock(&mc.lock); 4883 mc.from = NULL; 4884 mc.to = NULL; 4885 spin_unlock(&mc.lock); 4886 mem_cgroup_end_move(from); 4887} 4888 4889static int mem_cgroup_can_attach(struct cgroup_subsys *ss, 4890 struct cgroup *cgroup, 4891 struct task_struct *p, 4892 bool threadgroup) 4893{ 4894 int ret = 0; 4895 struct mem_cgroup *mem = mem_cgroup_from_cont(cgroup); 4896 4897 if (mem->move_charge_at_immigrate) { 4898 struct mm_struct *mm; 4899 struct mem_cgroup *from = mem_cgroup_from_task(p); 4900 4901 VM_BUG_ON(from == mem); 4902 4903 mm = get_task_mm(p); 4904 if (!mm) 4905 return 0; 4906 /* We move charges only when we move a owner of the mm */ 4907 if (mm->owner == p) { 4908 VM_BUG_ON(mc.from); 4909 VM_BUG_ON(mc.to); 4910 VM_BUG_ON(mc.precharge); 4911 VM_BUG_ON(mc.moved_charge); 4912 VM_BUG_ON(mc.moved_swap); 4913 mem_cgroup_start_move(from); 4914 spin_lock(&mc.lock); 4915 mc.from = from; 4916 mc.to = mem; 4917 spin_unlock(&mc.lock); 4918 /* We set mc.moving_task later */ 4919 4920 ret = mem_cgroup_precharge_mc(mm); 4921 if (ret) 4922 mem_cgroup_clear_mc(); 4923 } 4924 mmput(mm); 4925 } 4926 return ret; 4927} 4928 4929static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss, 4930 struct cgroup *cgroup, 4931 struct task_struct *p, 4932 bool threadgroup) 4933{ 4934 mem_cgroup_clear_mc(); 4935} 4936 4937static int mem_cgroup_move_charge_pte_range(pmd_t *pmd, 4938 unsigned long addr, unsigned long end, 4939 struct mm_walk *walk) 4940{ 4941 int ret = 0; 4942 struct vm_area_struct *vma = walk->private; 4943 pte_t *pte; 4944 spinlock_t *ptl; 4945 4946 split_huge_page_pmd(walk->mm, pmd); 4947retry: 4948 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 4949 for (; addr != end; addr += PAGE_SIZE) { 4950 pte_t ptent = *(pte++); 4951 union mc_target target; 4952 int type; 4953 struct page *page; 4954 struct page_cgroup *pc; 4955 swp_entry_t ent; 4956 4957 if (!mc.precharge) 4958 break; 4959 4960 type = is_target_pte_for_mc(vma, addr, ptent, &target); 4961 switch (type) { 4962 case MC_TARGET_PAGE: 4963 page = target.page; 4964 if (isolate_lru_page(page)) 4965 goto put; 4966 pc = lookup_page_cgroup(page); 4967 if (!mem_cgroup_move_account(page, pc, 4968 mc.from, mc.to, false, PAGE_SIZE)) { 4969 mc.precharge--; 4970 /* we uncharge from mc.from later. */ 4971 mc.moved_charge++; 4972 } 4973 putback_lru_page(page); 4974put: /* is_target_pte_for_mc() gets the page */ 4975 put_page(page); 4976 break; 4977 case MC_TARGET_SWAP: 4978 ent = target.ent; 4979 if (!mem_cgroup_move_swap_account(ent, 4980 mc.from, mc.to, false)) { 4981 mc.precharge--; 4982 /* we fixup refcnts and charges later. */ 4983 mc.moved_swap++; 4984 } 4985 break; 4986 default: 4987 break; 4988 } 4989 } 4990 pte_unmap_unlock(pte - 1, ptl); 4991 cond_resched(); 4992 4993 if (addr != end) { 4994 /* 4995 * We have consumed all precharges we got in can_attach(). 4996 * We try charge one by one, but don't do any additional 4997 * charges to mc.to if we have failed in charge once in attach() 4998 * phase. 4999 */ 5000 ret = mem_cgroup_do_precharge(1); 5001 if (!ret) 5002 goto retry; 5003 } 5004 5005 return ret; 5006} 5007 5008static void mem_cgroup_move_charge(struct mm_struct *mm) 5009{ 5010 struct vm_area_struct *vma; 5011 5012 lru_add_drain_all(); 5013retry: 5014 if (unlikely(!down_read_trylock(&mm->mmap_sem))) { 5015 /* 5016 * Someone who are holding the mmap_sem might be waiting in 5017 * waitq. So we cancel all extra charges, wake up all waiters, 5018 * and retry. Because we cancel precharges, we might not be able 5019 * to move enough charges, but moving charge is a best-effort 5020 * feature anyway, so it wouldn't be a big problem. 5021 */ 5022 __mem_cgroup_clear_mc(); 5023 cond_resched(); 5024 goto retry; 5025 } 5026 for (vma = mm->mmap; vma; vma = vma->vm_next) { 5027 int ret; 5028 struct mm_walk mem_cgroup_move_charge_walk = { 5029 .pmd_entry = mem_cgroup_move_charge_pte_range, 5030 .mm = mm, 5031 .private = vma, 5032 }; 5033 if (is_vm_hugetlb_page(vma)) 5034 continue; 5035 ret = walk_page_range(vma->vm_start, vma->vm_end, 5036 &mem_cgroup_move_charge_walk); 5037 if (ret) 5038 /* 5039 * means we have consumed all precharges and failed in 5040 * doing additional charge. Just abandon here. 5041 */ 5042 break; 5043 } 5044 up_read(&mm->mmap_sem); 5045} 5046 5047static void mem_cgroup_move_task(struct cgroup_subsys *ss, 5048 struct cgroup *cont, 5049 struct cgroup *old_cont, 5050 struct task_struct *p, 5051 bool threadgroup) 5052{ 5053 struct mm_struct *mm; 5054 5055 if (!mc.to) 5056 /* no need to move charge */ 5057 return; 5058 5059 mm = get_task_mm(p); 5060 if (mm) { 5061 mem_cgroup_move_charge(mm); 5062 mmput(mm); 5063 } 5064 mem_cgroup_clear_mc(); 5065} 5066#else /* !CONFIG_MMU */ 5067static int mem_cgroup_can_attach(struct cgroup_subsys *ss, 5068 struct cgroup *cgroup, 5069 struct task_struct *p, 5070 bool threadgroup) 5071{ 5072 return 0; 5073} 5074static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss, 5075 struct cgroup *cgroup, 5076 struct task_struct *p, 5077 bool threadgroup) 5078{ 5079} 5080static void mem_cgroup_move_task(struct cgroup_subsys *ss, 5081 struct cgroup *cont, 5082 struct cgroup *old_cont, 5083 struct task_struct *p, 5084 bool threadgroup) 5085{ 5086} 5087#endif 5088 5089struct cgroup_subsys mem_cgroup_subsys = { 5090 .name = "memory", 5091 .subsys_id = mem_cgroup_subsys_id, 5092 .create = mem_cgroup_create, 5093 .pre_destroy = mem_cgroup_pre_destroy, 5094 .destroy = mem_cgroup_destroy, 5095 .populate = mem_cgroup_populate, 5096 .can_attach = mem_cgroup_can_attach, 5097 .cancel_attach = mem_cgroup_cancel_attach, 5098 .attach = mem_cgroup_move_task, 5099 .early_init = 0, 5100 .use_id = 1, 5101}; 5102 5103#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP 5104static int __init enable_swap_account(char *s) 5105{ 5106 /* consider enabled if no parameter or 1 is given */ 5107 if (!(*s) || !strcmp(s, "=1")) 5108 really_do_swap_account = 1; 5109 else if (!strcmp(s, "=0")) 5110 really_do_swap_account = 0; 5111 return 1; 5112} 5113__setup("swapaccount", enable_swap_account); 5114 5115static int __init disable_swap_account(char *s) 5116{ 5117 printk_once("noswapaccount is deprecated and will be removed in 2.6.40. Use swapaccount=0 instead\n"); 5118 enable_swap_account("=0"); 5119 return 1; 5120} 5121__setup("noswapaccount", disable_swap_account); 5122#endif 5123