memcontrol.c revision fceda1bf498677501befc7da72fd2e4de7f18466
1/* memcontrol.c - Memory Controller 2 * 3 * Copyright IBM Corporation, 2007 4 * Author Balbir Singh <balbir@linux.vnet.ibm.com> 5 * 6 * Copyright 2007 OpenVZ SWsoft Inc 7 * Author: Pavel Emelianov <xemul@openvz.org> 8 * 9 * Memory thresholds 10 * Copyright (C) 2009 Nokia Corporation 11 * Author: Kirill A. Shutemov 12 * 13 * This program is free software; you can redistribute it and/or modify 14 * it under the terms of the GNU General Public License as published by 15 * the Free Software Foundation; either version 2 of the License, or 16 * (at your option) any later version. 17 * 18 * This program is distributed in the hope that it will be useful, 19 * but WITHOUT ANY WARRANTY; without even the implied warranty of 20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 21 * GNU General Public License for more details. 22 */ 23 24#include <linux/res_counter.h> 25#include <linux/memcontrol.h> 26#include <linux/cgroup.h> 27#include <linux/mm.h> 28#include <linux/hugetlb.h> 29#include <linux/pagemap.h> 30#include <linux/smp.h> 31#include <linux/page-flags.h> 32#include <linux/backing-dev.h> 33#include <linux/bit_spinlock.h> 34#include <linux/rcupdate.h> 35#include <linux/limits.h> 36#include <linux/mutex.h> 37#include <linux/rbtree.h> 38#include <linux/slab.h> 39#include <linux/swap.h> 40#include <linux/swapops.h> 41#include <linux/spinlock.h> 42#include <linux/eventfd.h> 43#include <linux/sort.h> 44#include <linux/fs.h> 45#include <linux/seq_file.h> 46#include <linux/vmalloc.h> 47#include <linux/mm_inline.h> 48#include <linux/page_cgroup.h> 49#include <linux/cpu.h> 50#include <linux/oom.h> 51#include "internal.h" 52 53#include <asm/uaccess.h> 54 55#include <trace/events/vmscan.h> 56 57struct cgroup_subsys mem_cgroup_subsys __read_mostly; 58#define MEM_CGROUP_RECLAIM_RETRIES 5 59struct mem_cgroup *root_mem_cgroup __read_mostly; 60 61#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP 62/* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */ 63int do_swap_account __read_mostly; 64 65/* for remember boot option*/ 66#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP_ENABLED 67static int really_do_swap_account __initdata = 1; 68#else 69static int really_do_swap_account __initdata = 0; 70#endif 71 72#else 73#define do_swap_account (0) 74#endif 75 76/* 77 * Per memcg event counter is incremented at every pagein/pageout. This counter 78 * is used for trigger some periodic events. This is straightforward and better 79 * than using jiffies etc. to handle periodic memcg event. 80 * 81 * These values will be used as !((event) & ((1 <<(thresh)) - 1)) 82 */ 83#define THRESHOLDS_EVENTS_THRESH (7) /* once in 128 */ 84#define SOFTLIMIT_EVENTS_THRESH (10) /* once in 1024 */ 85 86/* 87 * Statistics for memory cgroup. 88 */ 89enum mem_cgroup_stat_index { 90 /* 91 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss. 92 */ 93 MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */ 94 MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */ 95 MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */ 96 MEM_CGROUP_STAT_PGPGIN_COUNT, /* # of pages paged in */ 97 MEM_CGROUP_STAT_PGPGOUT_COUNT, /* # of pages paged out */ 98 MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */ 99 MEM_CGROUP_STAT_DATA, /* end of data requires synchronization */ 100 /* incremented at every pagein/pageout */ 101 MEM_CGROUP_EVENTS = MEM_CGROUP_STAT_DATA, 102 MEM_CGROUP_ON_MOVE, /* someone is moving account between groups */ 103 104 MEM_CGROUP_STAT_NSTATS, 105}; 106 107struct mem_cgroup_stat_cpu { 108 s64 count[MEM_CGROUP_STAT_NSTATS]; 109}; 110 111/* 112 * per-zone information in memory controller. 113 */ 114struct mem_cgroup_per_zone { 115 /* 116 * spin_lock to protect the per cgroup LRU 117 */ 118 struct list_head lists[NR_LRU_LISTS]; 119 unsigned long count[NR_LRU_LISTS]; 120 121 struct zone_reclaim_stat reclaim_stat; 122 struct rb_node tree_node; /* RB tree node */ 123 unsigned long long usage_in_excess;/* Set to the value by which */ 124 /* the soft limit is exceeded*/ 125 bool on_tree; 126 struct mem_cgroup *mem; /* Back pointer, we cannot */ 127 /* use container_of */ 128}; 129/* Macro for accessing counter */ 130#define MEM_CGROUP_ZSTAT(mz, idx) ((mz)->count[(idx)]) 131 132struct mem_cgroup_per_node { 133 struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES]; 134}; 135 136struct mem_cgroup_lru_info { 137 struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES]; 138}; 139 140/* 141 * Cgroups above their limits are maintained in a RB-Tree, independent of 142 * their hierarchy representation 143 */ 144 145struct mem_cgroup_tree_per_zone { 146 struct rb_root rb_root; 147 spinlock_t lock; 148}; 149 150struct mem_cgroup_tree_per_node { 151 struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES]; 152}; 153 154struct mem_cgroup_tree { 155 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES]; 156}; 157 158static struct mem_cgroup_tree soft_limit_tree __read_mostly; 159 160struct mem_cgroup_threshold { 161 struct eventfd_ctx *eventfd; 162 u64 threshold; 163}; 164 165/* For threshold */ 166struct mem_cgroup_threshold_ary { 167 /* An array index points to threshold just below usage. */ 168 int current_threshold; 169 /* Size of entries[] */ 170 unsigned int size; 171 /* Array of thresholds */ 172 struct mem_cgroup_threshold entries[0]; 173}; 174 175struct mem_cgroup_thresholds { 176 /* Primary thresholds array */ 177 struct mem_cgroup_threshold_ary *primary; 178 /* 179 * Spare threshold array. 180 * This is needed to make mem_cgroup_unregister_event() "never fail". 181 * It must be able to store at least primary->size - 1 entries. 182 */ 183 struct mem_cgroup_threshold_ary *spare; 184}; 185 186/* for OOM */ 187struct mem_cgroup_eventfd_list { 188 struct list_head list; 189 struct eventfd_ctx *eventfd; 190}; 191 192static void mem_cgroup_threshold(struct mem_cgroup *mem); 193static void mem_cgroup_oom_notify(struct mem_cgroup *mem); 194 195/* 196 * The memory controller data structure. The memory controller controls both 197 * page cache and RSS per cgroup. We would eventually like to provide 198 * statistics based on the statistics developed by Rik Van Riel for clock-pro, 199 * to help the administrator determine what knobs to tune. 200 * 201 * TODO: Add a water mark for the memory controller. Reclaim will begin when 202 * we hit the water mark. May be even add a low water mark, such that 203 * no reclaim occurs from a cgroup at it's low water mark, this is 204 * a feature that will be implemented much later in the future. 205 */ 206struct mem_cgroup { 207 struct cgroup_subsys_state css; 208 /* 209 * the counter to account for memory usage 210 */ 211 struct res_counter res; 212 /* 213 * the counter to account for mem+swap usage. 214 */ 215 struct res_counter memsw; 216 /* 217 * Per cgroup active and inactive list, similar to the 218 * per zone LRU lists. 219 */ 220 struct mem_cgroup_lru_info info; 221 222 /* 223 protect against reclaim related member. 224 */ 225 spinlock_t reclaim_param_lock; 226 227 /* 228 * While reclaiming in a hierarchy, we cache the last child we 229 * reclaimed from. 230 */ 231 int last_scanned_child; 232 /* 233 * Should the accounting and control be hierarchical, per subtree? 234 */ 235 bool use_hierarchy; 236 atomic_t oom_lock; 237 atomic_t refcnt; 238 239 unsigned int swappiness; 240 /* OOM-Killer disable */ 241 int oom_kill_disable; 242 243 /* set when res.limit == memsw.limit */ 244 bool memsw_is_minimum; 245 246 /* protect arrays of thresholds */ 247 struct mutex thresholds_lock; 248 249 /* thresholds for memory usage. RCU-protected */ 250 struct mem_cgroup_thresholds thresholds; 251 252 /* thresholds for mem+swap usage. RCU-protected */ 253 struct mem_cgroup_thresholds memsw_thresholds; 254 255 /* For oom notifier event fd */ 256 struct list_head oom_notify; 257 258 /* 259 * Should we move charges of a task when a task is moved into this 260 * mem_cgroup ? And what type of charges should we move ? 261 */ 262 unsigned long move_charge_at_immigrate; 263 /* 264 * percpu counter. 265 */ 266 struct mem_cgroup_stat_cpu *stat; 267 /* 268 * used when a cpu is offlined or other synchronizations 269 * See mem_cgroup_read_stat(). 270 */ 271 struct mem_cgroup_stat_cpu nocpu_base; 272 spinlock_t pcp_counter_lock; 273}; 274 275/* Stuffs for move charges at task migration. */ 276/* 277 * Types of charges to be moved. "move_charge_at_immitgrate" is treated as a 278 * left-shifted bitmap of these types. 279 */ 280enum move_type { 281 MOVE_CHARGE_TYPE_ANON, /* private anonymous page and swap of it */ 282 MOVE_CHARGE_TYPE_FILE, /* file page(including tmpfs) and swap of it */ 283 NR_MOVE_TYPE, 284}; 285 286/* "mc" and its members are protected by cgroup_mutex */ 287static struct move_charge_struct { 288 spinlock_t lock; /* for from, to */ 289 struct mem_cgroup *from; 290 struct mem_cgroup *to; 291 unsigned long precharge; 292 unsigned long moved_charge; 293 unsigned long moved_swap; 294 struct task_struct *moving_task; /* a task moving charges */ 295 wait_queue_head_t waitq; /* a waitq for other context */ 296} mc = { 297 .lock = __SPIN_LOCK_UNLOCKED(mc.lock), 298 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq), 299}; 300 301static bool move_anon(void) 302{ 303 return test_bit(MOVE_CHARGE_TYPE_ANON, 304 &mc.to->move_charge_at_immigrate); 305} 306 307static bool move_file(void) 308{ 309 return test_bit(MOVE_CHARGE_TYPE_FILE, 310 &mc.to->move_charge_at_immigrate); 311} 312 313/* 314 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft 315 * limit reclaim to prevent infinite loops, if they ever occur. 316 */ 317#define MEM_CGROUP_MAX_RECLAIM_LOOPS (100) 318#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS (2) 319 320enum charge_type { 321 MEM_CGROUP_CHARGE_TYPE_CACHE = 0, 322 MEM_CGROUP_CHARGE_TYPE_MAPPED, 323 MEM_CGROUP_CHARGE_TYPE_SHMEM, /* used by page migration of shmem */ 324 MEM_CGROUP_CHARGE_TYPE_FORCE, /* used by force_empty */ 325 MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */ 326 MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */ 327 NR_CHARGE_TYPE, 328}; 329 330/* only for here (for easy reading.) */ 331#define PCGF_CACHE (1UL << PCG_CACHE) 332#define PCGF_USED (1UL << PCG_USED) 333#define PCGF_LOCK (1UL << PCG_LOCK) 334/* Not used, but added here for completeness */ 335#define PCGF_ACCT (1UL << PCG_ACCT) 336 337/* for encoding cft->private value on file */ 338#define _MEM (0) 339#define _MEMSWAP (1) 340#define _OOM_TYPE (2) 341#define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val)) 342#define MEMFILE_TYPE(val) (((val) >> 16) & 0xffff) 343#define MEMFILE_ATTR(val) ((val) & 0xffff) 344/* Used for OOM nofiier */ 345#define OOM_CONTROL (0) 346 347/* 348 * Reclaim flags for mem_cgroup_hierarchical_reclaim 349 */ 350#define MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0 351#define MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT) 352#define MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1 353#define MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT) 354#define MEM_CGROUP_RECLAIM_SOFT_BIT 0x2 355#define MEM_CGROUP_RECLAIM_SOFT (1 << MEM_CGROUP_RECLAIM_SOFT_BIT) 356 357static void mem_cgroup_get(struct mem_cgroup *mem); 358static void mem_cgroup_put(struct mem_cgroup *mem); 359static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem); 360static void drain_all_stock_async(void); 361 362static struct mem_cgroup_per_zone * 363mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid) 364{ 365 return &mem->info.nodeinfo[nid]->zoneinfo[zid]; 366} 367 368struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem) 369{ 370 return &mem->css; 371} 372 373static struct mem_cgroup_per_zone * 374page_cgroup_zoneinfo(struct page_cgroup *pc) 375{ 376 struct mem_cgroup *mem = pc->mem_cgroup; 377 int nid = page_cgroup_nid(pc); 378 int zid = page_cgroup_zid(pc); 379 380 if (!mem) 381 return NULL; 382 383 return mem_cgroup_zoneinfo(mem, nid, zid); 384} 385 386static struct mem_cgroup_tree_per_zone * 387soft_limit_tree_node_zone(int nid, int zid) 388{ 389 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid]; 390} 391 392static struct mem_cgroup_tree_per_zone * 393soft_limit_tree_from_page(struct page *page) 394{ 395 int nid = page_to_nid(page); 396 int zid = page_zonenum(page); 397 398 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid]; 399} 400 401static void 402__mem_cgroup_insert_exceeded(struct mem_cgroup *mem, 403 struct mem_cgroup_per_zone *mz, 404 struct mem_cgroup_tree_per_zone *mctz, 405 unsigned long long new_usage_in_excess) 406{ 407 struct rb_node **p = &mctz->rb_root.rb_node; 408 struct rb_node *parent = NULL; 409 struct mem_cgroup_per_zone *mz_node; 410 411 if (mz->on_tree) 412 return; 413 414 mz->usage_in_excess = new_usage_in_excess; 415 if (!mz->usage_in_excess) 416 return; 417 while (*p) { 418 parent = *p; 419 mz_node = rb_entry(parent, struct mem_cgroup_per_zone, 420 tree_node); 421 if (mz->usage_in_excess < mz_node->usage_in_excess) 422 p = &(*p)->rb_left; 423 /* 424 * We can't avoid mem cgroups that are over their soft 425 * limit by the same amount 426 */ 427 else if (mz->usage_in_excess >= mz_node->usage_in_excess) 428 p = &(*p)->rb_right; 429 } 430 rb_link_node(&mz->tree_node, parent, p); 431 rb_insert_color(&mz->tree_node, &mctz->rb_root); 432 mz->on_tree = true; 433} 434 435static void 436__mem_cgroup_remove_exceeded(struct mem_cgroup *mem, 437 struct mem_cgroup_per_zone *mz, 438 struct mem_cgroup_tree_per_zone *mctz) 439{ 440 if (!mz->on_tree) 441 return; 442 rb_erase(&mz->tree_node, &mctz->rb_root); 443 mz->on_tree = false; 444} 445 446static void 447mem_cgroup_remove_exceeded(struct mem_cgroup *mem, 448 struct mem_cgroup_per_zone *mz, 449 struct mem_cgroup_tree_per_zone *mctz) 450{ 451 spin_lock(&mctz->lock); 452 __mem_cgroup_remove_exceeded(mem, mz, mctz); 453 spin_unlock(&mctz->lock); 454} 455 456 457static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page) 458{ 459 unsigned long long excess; 460 struct mem_cgroup_per_zone *mz; 461 struct mem_cgroup_tree_per_zone *mctz; 462 int nid = page_to_nid(page); 463 int zid = page_zonenum(page); 464 mctz = soft_limit_tree_from_page(page); 465 466 /* 467 * Necessary to update all ancestors when hierarchy is used. 468 * because their event counter is not touched. 469 */ 470 for (; mem; mem = parent_mem_cgroup(mem)) { 471 mz = mem_cgroup_zoneinfo(mem, nid, zid); 472 excess = res_counter_soft_limit_excess(&mem->res); 473 /* 474 * We have to update the tree if mz is on RB-tree or 475 * mem is over its softlimit. 476 */ 477 if (excess || mz->on_tree) { 478 spin_lock(&mctz->lock); 479 /* if on-tree, remove it */ 480 if (mz->on_tree) 481 __mem_cgroup_remove_exceeded(mem, mz, mctz); 482 /* 483 * Insert again. mz->usage_in_excess will be updated. 484 * If excess is 0, no tree ops. 485 */ 486 __mem_cgroup_insert_exceeded(mem, mz, mctz, excess); 487 spin_unlock(&mctz->lock); 488 } 489 } 490} 491 492static void mem_cgroup_remove_from_trees(struct mem_cgroup *mem) 493{ 494 int node, zone; 495 struct mem_cgroup_per_zone *mz; 496 struct mem_cgroup_tree_per_zone *mctz; 497 498 for_each_node_state(node, N_POSSIBLE) { 499 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 500 mz = mem_cgroup_zoneinfo(mem, node, zone); 501 mctz = soft_limit_tree_node_zone(node, zone); 502 mem_cgroup_remove_exceeded(mem, mz, mctz); 503 } 504 } 505} 506 507static inline unsigned long mem_cgroup_get_excess(struct mem_cgroup *mem) 508{ 509 return res_counter_soft_limit_excess(&mem->res) >> PAGE_SHIFT; 510} 511 512static struct mem_cgroup_per_zone * 513__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz) 514{ 515 struct rb_node *rightmost = NULL; 516 struct mem_cgroup_per_zone *mz; 517 518retry: 519 mz = NULL; 520 rightmost = rb_last(&mctz->rb_root); 521 if (!rightmost) 522 goto done; /* Nothing to reclaim from */ 523 524 mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node); 525 /* 526 * Remove the node now but someone else can add it back, 527 * we will to add it back at the end of reclaim to its correct 528 * position in the tree. 529 */ 530 __mem_cgroup_remove_exceeded(mz->mem, mz, mctz); 531 if (!res_counter_soft_limit_excess(&mz->mem->res) || 532 !css_tryget(&mz->mem->css)) 533 goto retry; 534done: 535 return mz; 536} 537 538static struct mem_cgroup_per_zone * 539mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz) 540{ 541 struct mem_cgroup_per_zone *mz; 542 543 spin_lock(&mctz->lock); 544 mz = __mem_cgroup_largest_soft_limit_node(mctz); 545 spin_unlock(&mctz->lock); 546 return mz; 547} 548 549/* 550 * Implementation Note: reading percpu statistics for memcg. 551 * 552 * Both of vmstat[] and percpu_counter has threshold and do periodic 553 * synchronization to implement "quick" read. There are trade-off between 554 * reading cost and precision of value. Then, we may have a chance to implement 555 * a periodic synchronizion of counter in memcg's counter. 556 * 557 * But this _read() function is used for user interface now. The user accounts 558 * memory usage by memory cgroup and he _always_ requires exact value because 559 * he accounts memory. Even if we provide quick-and-fuzzy read, we always 560 * have to visit all online cpus and make sum. So, for now, unnecessary 561 * synchronization is not implemented. (just implemented for cpu hotplug) 562 * 563 * If there are kernel internal actions which can make use of some not-exact 564 * value, and reading all cpu value can be performance bottleneck in some 565 * common workload, threashold and synchonization as vmstat[] should be 566 * implemented. 567 */ 568static s64 mem_cgroup_read_stat(struct mem_cgroup *mem, 569 enum mem_cgroup_stat_index idx) 570{ 571 int cpu; 572 s64 val = 0; 573 574 get_online_cpus(); 575 for_each_online_cpu(cpu) 576 val += per_cpu(mem->stat->count[idx], cpu); 577#ifdef CONFIG_HOTPLUG_CPU 578 spin_lock(&mem->pcp_counter_lock); 579 val += mem->nocpu_base.count[idx]; 580 spin_unlock(&mem->pcp_counter_lock); 581#endif 582 put_online_cpus(); 583 return val; 584} 585 586static s64 mem_cgroup_local_usage(struct mem_cgroup *mem) 587{ 588 s64 ret; 589 590 ret = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS); 591 ret += mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE); 592 return ret; 593} 594 595static void mem_cgroup_swap_statistics(struct mem_cgroup *mem, 596 bool charge) 597{ 598 int val = (charge) ? 1 : -1; 599 this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_SWAPOUT], val); 600} 601 602static void mem_cgroup_charge_statistics(struct mem_cgroup *mem, 603 bool file, int nr_pages) 604{ 605 preempt_disable(); 606 607 if (file) 608 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_CACHE], nr_pages); 609 else 610 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_RSS], nr_pages); 611 612 /* pagein of a big page is an event. So, ignore page size */ 613 if (nr_pages > 0) 614 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGIN_COUNT]); 615 else 616 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGOUT_COUNT]); 617 618 __this_cpu_add(mem->stat->count[MEM_CGROUP_EVENTS], nr_pages); 619 620 preempt_enable(); 621} 622 623static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem, 624 enum lru_list idx) 625{ 626 int nid, zid; 627 struct mem_cgroup_per_zone *mz; 628 u64 total = 0; 629 630 for_each_online_node(nid) 631 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 632 mz = mem_cgroup_zoneinfo(mem, nid, zid); 633 total += MEM_CGROUP_ZSTAT(mz, idx); 634 } 635 return total; 636} 637 638static bool __memcg_event_check(struct mem_cgroup *mem, int event_mask_shift) 639{ 640 s64 val; 641 642 val = this_cpu_read(mem->stat->count[MEM_CGROUP_EVENTS]); 643 644 return !(val & ((1 << event_mask_shift) - 1)); 645} 646 647/* 648 * Check events in order. 649 * 650 */ 651static void memcg_check_events(struct mem_cgroup *mem, struct page *page) 652{ 653 /* threshold event is triggered in finer grain than soft limit */ 654 if (unlikely(__memcg_event_check(mem, THRESHOLDS_EVENTS_THRESH))) { 655 mem_cgroup_threshold(mem); 656 if (unlikely(__memcg_event_check(mem, SOFTLIMIT_EVENTS_THRESH))) 657 mem_cgroup_update_tree(mem, page); 658 } 659} 660 661static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont) 662{ 663 return container_of(cgroup_subsys_state(cont, 664 mem_cgroup_subsys_id), struct mem_cgroup, 665 css); 666} 667 668struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) 669{ 670 /* 671 * mm_update_next_owner() may clear mm->owner to NULL 672 * if it races with swapoff, page migration, etc. 673 * So this can be called with p == NULL. 674 */ 675 if (unlikely(!p)) 676 return NULL; 677 678 return container_of(task_subsys_state(p, mem_cgroup_subsys_id), 679 struct mem_cgroup, css); 680} 681 682static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm) 683{ 684 struct mem_cgroup *mem = NULL; 685 686 if (!mm) 687 return NULL; 688 /* 689 * Because we have no locks, mm->owner's may be being moved to other 690 * cgroup. We use css_tryget() here even if this looks 691 * pessimistic (rather than adding locks here). 692 */ 693 rcu_read_lock(); 694 do { 695 mem = mem_cgroup_from_task(rcu_dereference(mm->owner)); 696 if (unlikely(!mem)) 697 break; 698 } while (!css_tryget(&mem->css)); 699 rcu_read_unlock(); 700 return mem; 701} 702 703/* The caller has to guarantee "mem" exists before calling this */ 704static struct mem_cgroup *mem_cgroup_start_loop(struct mem_cgroup *mem) 705{ 706 struct cgroup_subsys_state *css; 707 int found; 708 709 if (!mem) /* ROOT cgroup has the smallest ID */ 710 return root_mem_cgroup; /*css_put/get against root is ignored*/ 711 if (!mem->use_hierarchy) { 712 if (css_tryget(&mem->css)) 713 return mem; 714 return NULL; 715 } 716 rcu_read_lock(); 717 /* 718 * searching a memory cgroup which has the smallest ID under given 719 * ROOT cgroup. (ID >= 1) 720 */ 721 css = css_get_next(&mem_cgroup_subsys, 1, &mem->css, &found); 722 if (css && css_tryget(css)) 723 mem = container_of(css, struct mem_cgroup, css); 724 else 725 mem = NULL; 726 rcu_read_unlock(); 727 return mem; 728} 729 730static struct mem_cgroup *mem_cgroup_get_next(struct mem_cgroup *iter, 731 struct mem_cgroup *root, 732 bool cond) 733{ 734 int nextid = css_id(&iter->css) + 1; 735 int found; 736 int hierarchy_used; 737 struct cgroup_subsys_state *css; 738 739 hierarchy_used = iter->use_hierarchy; 740 741 css_put(&iter->css); 742 /* If no ROOT, walk all, ignore hierarchy */ 743 if (!cond || (root && !hierarchy_used)) 744 return NULL; 745 746 if (!root) 747 root = root_mem_cgroup; 748 749 do { 750 iter = NULL; 751 rcu_read_lock(); 752 753 css = css_get_next(&mem_cgroup_subsys, nextid, 754 &root->css, &found); 755 if (css && css_tryget(css)) 756 iter = container_of(css, struct mem_cgroup, css); 757 rcu_read_unlock(); 758 /* If css is NULL, no more cgroups will be found */ 759 nextid = found + 1; 760 } while (css && !iter); 761 762 return iter; 763} 764/* 765 * for_eacn_mem_cgroup_tree() for visiting all cgroup under tree. Please 766 * be careful that "break" loop is not allowed. We have reference count. 767 * Instead of that modify "cond" to be false and "continue" to exit the loop. 768 */ 769#define for_each_mem_cgroup_tree_cond(iter, root, cond) \ 770 for (iter = mem_cgroup_start_loop(root);\ 771 iter != NULL;\ 772 iter = mem_cgroup_get_next(iter, root, cond)) 773 774#define for_each_mem_cgroup_tree(iter, root) \ 775 for_each_mem_cgroup_tree_cond(iter, root, true) 776 777#define for_each_mem_cgroup_all(iter) \ 778 for_each_mem_cgroup_tree_cond(iter, NULL, true) 779 780 781static inline bool mem_cgroup_is_root(struct mem_cgroup *mem) 782{ 783 return (mem == root_mem_cgroup); 784} 785 786/* 787 * Following LRU functions are allowed to be used without PCG_LOCK. 788 * Operations are called by routine of global LRU independently from memcg. 789 * What we have to take care of here is validness of pc->mem_cgroup. 790 * 791 * Changes to pc->mem_cgroup happens when 792 * 1. charge 793 * 2. moving account 794 * In typical case, "charge" is done before add-to-lru. Exception is SwapCache. 795 * It is added to LRU before charge. 796 * If PCG_USED bit is not set, page_cgroup is not added to this private LRU. 797 * When moving account, the page is not on LRU. It's isolated. 798 */ 799 800void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru) 801{ 802 struct page_cgroup *pc; 803 struct mem_cgroup_per_zone *mz; 804 805 if (mem_cgroup_disabled()) 806 return; 807 pc = lookup_page_cgroup(page); 808 /* can happen while we handle swapcache. */ 809 if (!TestClearPageCgroupAcctLRU(pc)) 810 return; 811 VM_BUG_ON(!pc->mem_cgroup); 812 /* 813 * We don't check PCG_USED bit. It's cleared when the "page" is finally 814 * removed from global LRU. 815 */ 816 mz = page_cgroup_zoneinfo(pc); 817 /* huge page split is done under lru_lock. so, we have no races. */ 818 MEM_CGROUP_ZSTAT(mz, lru) -= 1 << compound_order(page); 819 if (mem_cgroup_is_root(pc->mem_cgroup)) 820 return; 821 VM_BUG_ON(list_empty(&pc->lru)); 822 list_del_init(&pc->lru); 823} 824 825void mem_cgroup_del_lru(struct page *page) 826{ 827 mem_cgroup_del_lru_list(page, page_lru(page)); 828} 829 830void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru) 831{ 832 struct mem_cgroup_per_zone *mz; 833 struct page_cgroup *pc; 834 835 if (mem_cgroup_disabled()) 836 return; 837 838 pc = lookup_page_cgroup(page); 839 /* unused or root page is not rotated. */ 840 if (!PageCgroupUsed(pc)) 841 return; 842 /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */ 843 smp_rmb(); 844 if (mem_cgroup_is_root(pc->mem_cgroup)) 845 return; 846 mz = page_cgroup_zoneinfo(pc); 847 list_move(&pc->lru, &mz->lists[lru]); 848} 849 850void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru) 851{ 852 struct page_cgroup *pc; 853 struct mem_cgroup_per_zone *mz; 854 855 if (mem_cgroup_disabled()) 856 return; 857 pc = lookup_page_cgroup(page); 858 VM_BUG_ON(PageCgroupAcctLRU(pc)); 859 if (!PageCgroupUsed(pc)) 860 return; 861 /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */ 862 smp_rmb(); 863 mz = page_cgroup_zoneinfo(pc); 864 /* huge page split is done under lru_lock. so, we have no races. */ 865 MEM_CGROUP_ZSTAT(mz, lru) += 1 << compound_order(page); 866 SetPageCgroupAcctLRU(pc); 867 if (mem_cgroup_is_root(pc->mem_cgroup)) 868 return; 869 list_add(&pc->lru, &mz->lists[lru]); 870} 871 872/* 873 * At handling SwapCache, pc->mem_cgroup may be changed while it's linked to 874 * lru because the page may.be reused after it's fully uncharged (because of 875 * SwapCache behavior).To handle that, unlink page_cgroup from LRU when charge 876 * it again. This function is only used to charge SwapCache. It's done under 877 * lock_page and expected that zone->lru_lock is never held. 878 */ 879static void mem_cgroup_lru_del_before_commit_swapcache(struct page *page) 880{ 881 unsigned long flags; 882 struct zone *zone = page_zone(page); 883 struct page_cgroup *pc = lookup_page_cgroup(page); 884 885 spin_lock_irqsave(&zone->lru_lock, flags); 886 /* 887 * Forget old LRU when this page_cgroup is *not* used. This Used bit 888 * is guarded by lock_page() because the page is SwapCache. 889 */ 890 if (!PageCgroupUsed(pc)) 891 mem_cgroup_del_lru_list(page, page_lru(page)); 892 spin_unlock_irqrestore(&zone->lru_lock, flags); 893} 894 895static void mem_cgroup_lru_add_after_commit_swapcache(struct page *page) 896{ 897 unsigned long flags; 898 struct zone *zone = page_zone(page); 899 struct page_cgroup *pc = lookup_page_cgroup(page); 900 901 spin_lock_irqsave(&zone->lru_lock, flags); 902 /* link when the page is linked to LRU but page_cgroup isn't */ 903 if (PageLRU(page) && !PageCgroupAcctLRU(pc)) 904 mem_cgroup_add_lru_list(page, page_lru(page)); 905 spin_unlock_irqrestore(&zone->lru_lock, flags); 906} 907 908 909void mem_cgroup_move_lists(struct page *page, 910 enum lru_list from, enum lru_list to) 911{ 912 if (mem_cgroup_disabled()) 913 return; 914 mem_cgroup_del_lru_list(page, from); 915 mem_cgroup_add_lru_list(page, to); 916} 917 918int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem) 919{ 920 int ret; 921 struct mem_cgroup *curr = NULL; 922 struct task_struct *p; 923 924 p = find_lock_task_mm(task); 925 if (!p) 926 return 0; 927 curr = try_get_mem_cgroup_from_mm(p->mm); 928 task_unlock(p); 929 if (!curr) 930 return 0; 931 /* 932 * We should check use_hierarchy of "mem" not "curr". Because checking 933 * use_hierarchy of "curr" here make this function true if hierarchy is 934 * enabled in "curr" and "curr" is a child of "mem" in *cgroup* 935 * hierarchy(even if use_hierarchy is disabled in "mem"). 936 */ 937 if (mem->use_hierarchy) 938 ret = css_is_ancestor(&curr->css, &mem->css); 939 else 940 ret = (curr == mem); 941 css_put(&curr->css); 942 return ret; 943} 944 945static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_pages) 946{ 947 unsigned long active; 948 unsigned long inactive; 949 unsigned long gb; 950 unsigned long inactive_ratio; 951 952 inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON); 953 active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON); 954 955 gb = (inactive + active) >> (30 - PAGE_SHIFT); 956 if (gb) 957 inactive_ratio = int_sqrt(10 * gb); 958 else 959 inactive_ratio = 1; 960 961 if (present_pages) { 962 present_pages[0] = inactive; 963 present_pages[1] = active; 964 } 965 966 return inactive_ratio; 967} 968 969int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg) 970{ 971 unsigned long active; 972 unsigned long inactive; 973 unsigned long present_pages[2]; 974 unsigned long inactive_ratio; 975 976 inactive_ratio = calc_inactive_ratio(memcg, present_pages); 977 978 inactive = present_pages[0]; 979 active = present_pages[1]; 980 981 if (inactive * inactive_ratio < active) 982 return 1; 983 984 return 0; 985} 986 987int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg) 988{ 989 unsigned long active; 990 unsigned long inactive; 991 992 inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE); 993 active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE); 994 995 return (active > inactive); 996} 997 998unsigned long mem_cgroup_zone_nr_pages(struct mem_cgroup *memcg, 999 struct zone *zone, 1000 enum lru_list lru) 1001{ 1002 int nid = zone_to_nid(zone); 1003 int zid = zone_idx(zone); 1004 struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid); 1005 1006 return MEM_CGROUP_ZSTAT(mz, lru); 1007} 1008 1009struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg, 1010 struct zone *zone) 1011{ 1012 int nid = zone_to_nid(zone); 1013 int zid = zone_idx(zone); 1014 struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid); 1015 1016 return &mz->reclaim_stat; 1017} 1018 1019struct zone_reclaim_stat * 1020mem_cgroup_get_reclaim_stat_from_page(struct page *page) 1021{ 1022 struct page_cgroup *pc; 1023 struct mem_cgroup_per_zone *mz; 1024 1025 if (mem_cgroup_disabled()) 1026 return NULL; 1027 1028 pc = lookup_page_cgroup(page); 1029 if (!PageCgroupUsed(pc)) 1030 return NULL; 1031 /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */ 1032 smp_rmb(); 1033 mz = page_cgroup_zoneinfo(pc); 1034 if (!mz) 1035 return NULL; 1036 1037 return &mz->reclaim_stat; 1038} 1039 1040unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan, 1041 struct list_head *dst, 1042 unsigned long *scanned, int order, 1043 int mode, struct zone *z, 1044 struct mem_cgroup *mem_cont, 1045 int active, int file) 1046{ 1047 unsigned long nr_taken = 0; 1048 struct page *page; 1049 unsigned long scan; 1050 LIST_HEAD(pc_list); 1051 struct list_head *src; 1052 struct page_cgroup *pc, *tmp; 1053 int nid = zone_to_nid(z); 1054 int zid = zone_idx(z); 1055 struct mem_cgroup_per_zone *mz; 1056 int lru = LRU_FILE * file + active; 1057 int ret; 1058 1059 BUG_ON(!mem_cont); 1060 mz = mem_cgroup_zoneinfo(mem_cont, nid, zid); 1061 src = &mz->lists[lru]; 1062 1063 scan = 0; 1064 list_for_each_entry_safe_reverse(pc, tmp, src, lru) { 1065 if (scan >= nr_to_scan) 1066 break; 1067 1068 page = pc->page; 1069 if (unlikely(!PageCgroupUsed(pc))) 1070 continue; 1071 if (unlikely(!PageLRU(page))) 1072 continue; 1073 1074 scan++; 1075 ret = __isolate_lru_page(page, mode, file); 1076 switch (ret) { 1077 case 0: 1078 list_move(&page->lru, dst); 1079 mem_cgroup_del_lru(page); 1080 nr_taken += hpage_nr_pages(page); 1081 break; 1082 case -EBUSY: 1083 /* we don't affect global LRU but rotate in our LRU */ 1084 mem_cgroup_rotate_lru_list(page, page_lru(page)); 1085 break; 1086 default: 1087 break; 1088 } 1089 } 1090 1091 *scanned = scan; 1092 1093 trace_mm_vmscan_memcg_isolate(0, nr_to_scan, scan, nr_taken, 1094 0, 0, 0, mode); 1095 1096 return nr_taken; 1097} 1098 1099#define mem_cgroup_from_res_counter(counter, member) \ 1100 container_of(counter, struct mem_cgroup, member) 1101 1102static bool mem_cgroup_check_under_limit(struct mem_cgroup *mem) 1103{ 1104 if (do_swap_account) { 1105 if (res_counter_check_under_limit(&mem->res) && 1106 res_counter_check_under_limit(&mem->memsw)) 1107 return true; 1108 } else 1109 if (res_counter_check_under_limit(&mem->res)) 1110 return true; 1111 return false; 1112} 1113 1114static unsigned int get_swappiness(struct mem_cgroup *memcg) 1115{ 1116 struct cgroup *cgrp = memcg->css.cgroup; 1117 unsigned int swappiness; 1118 1119 /* root ? */ 1120 if (cgrp->parent == NULL) 1121 return vm_swappiness; 1122 1123 spin_lock(&memcg->reclaim_param_lock); 1124 swappiness = memcg->swappiness; 1125 spin_unlock(&memcg->reclaim_param_lock); 1126 1127 return swappiness; 1128} 1129 1130static void mem_cgroup_start_move(struct mem_cgroup *mem) 1131{ 1132 int cpu; 1133 1134 get_online_cpus(); 1135 spin_lock(&mem->pcp_counter_lock); 1136 for_each_online_cpu(cpu) 1137 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) += 1; 1138 mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] += 1; 1139 spin_unlock(&mem->pcp_counter_lock); 1140 put_online_cpus(); 1141 1142 synchronize_rcu(); 1143} 1144 1145static void mem_cgroup_end_move(struct mem_cgroup *mem) 1146{ 1147 int cpu; 1148 1149 if (!mem) 1150 return; 1151 get_online_cpus(); 1152 spin_lock(&mem->pcp_counter_lock); 1153 for_each_online_cpu(cpu) 1154 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) -= 1; 1155 mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] -= 1; 1156 spin_unlock(&mem->pcp_counter_lock); 1157 put_online_cpus(); 1158} 1159/* 1160 * 2 routines for checking "mem" is under move_account() or not. 1161 * 1162 * mem_cgroup_stealed() - checking a cgroup is mc.from or not. This is used 1163 * for avoiding race in accounting. If true, 1164 * pc->mem_cgroup may be overwritten. 1165 * 1166 * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or 1167 * under hierarchy of moving cgroups. This is for 1168 * waiting at hith-memory prressure caused by "move". 1169 */ 1170 1171static bool mem_cgroup_stealed(struct mem_cgroup *mem) 1172{ 1173 VM_BUG_ON(!rcu_read_lock_held()); 1174 return this_cpu_read(mem->stat->count[MEM_CGROUP_ON_MOVE]) > 0; 1175} 1176 1177static bool mem_cgroup_under_move(struct mem_cgroup *mem) 1178{ 1179 struct mem_cgroup *from; 1180 struct mem_cgroup *to; 1181 bool ret = false; 1182 /* 1183 * Unlike task_move routines, we access mc.to, mc.from not under 1184 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead. 1185 */ 1186 spin_lock(&mc.lock); 1187 from = mc.from; 1188 to = mc.to; 1189 if (!from) 1190 goto unlock; 1191 if (from == mem || to == mem 1192 || (mem->use_hierarchy && css_is_ancestor(&from->css, &mem->css)) 1193 || (mem->use_hierarchy && css_is_ancestor(&to->css, &mem->css))) 1194 ret = true; 1195unlock: 1196 spin_unlock(&mc.lock); 1197 return ret; 1198} 1199 1200static bool mem_cgroup_wait_acct_move(struct mem_cgroup *mem) 1201{ 1202 if (mc.moving_task && current != mc.moving_task) { 1203 if (mem_cgroup_under_move(mem)) { 1204 DEFINE_WAIT(wait); 1205 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE); 1206 /* moving charge context might have finished. */ 1207 if (mc.moving_task) 1208 schedule(); 1209 finish_wait(&mc.waitq, &wait); 1210 return true; 1211 } 1212 } 1213 return false; 1214} 1215 1216/** 1217 * mem_cgroup_print_oom_info: Called from OOM with tasklist_lock held in read mode. 1218 * @memcg: The memory cgroup that went over limit 1219 * @p: Task that is going to be killed 1220 * 1221 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is 1222 * enabled 1223 */ 1224void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p) 1225{ 1226 struct cgroup *task_cgrp; 1227 struct cgroup *mem_cgrp; 1228 /* 1229 * Need a buffer in BSS, can't rely on allocations. The code relies 1230 * on the assumption that OOM is serialized for memory controller. 1231 * If this assumption is broken, revisit this code. 1232 */ 1233 static char memcg_name[PATH_MAX]; 1234 int ret; 1235 1236 if (!memcg || !p) 1237 return; 1238 1239 1240 rcu_read_lock(); 1241 1242 mem_cgrp = memcg->css.cgroup; 1243 task_cgrp = task_cgroup(p, mem_cgroup_subsys_id); 1244 1245 ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX); 1246 if (ret < 0) { 1247 /* 1248 * Unfortunately, we are unable to convert to a useful name 1249 * But we'll still print out the usage information 1250 */ 1251 rcu_read_unlock(); 1252 goto done; 1253 } 1254 rcu_read_unlock(); 1255 1256 printk(KERN_INFO "Task in %s killed", memcg_name); 1257 1258 rcu_read_lock(); 1259 ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX); 1260 if (ret < 0) { 1261 rcu_read_unlock(); 1262 goto done; 1263 } 1264 rcu_read_unlock(); 1265 1266 /* 1267 * Continues from above, so we don't need an KERN_ level 1268 */ 1269 printk(KERN_CONT " as a result of limit of %s\n", memcg_name); 1270done: 1271 1272 printk(KERN_INFO "memory: usage %llukB, limit %llukB, failcnt %llu\n", 1273 res_counter_read_u64(&memcg->res, RES_USAGE) >> 10, 1274 res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10, 1275 res_counter_read_u64(&memcg->res, RES_FAILCNT)); 1276 printk(KERN_INFO "memory+swap: usage %llukB, limit %llukB, " 1277 "failcnt %llu\n", 1278 res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10, 1279 res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10, 1280 res_counter_read_u64(&memcg->memsw, RES_FAILCNT)); 1281} 1282 1283/* 1284 * This function returns the number of memcg under hierarchy tree. Returns 1285 * 1(self count) if no children. 1286 */ 1287static int mem_cgroup_count_children(struct mem_cgroup *mem) 1288{ 1289 int num = 0; 1290 struct mem_cgroup *iter; 1291 1292 for_each_mem_cgroup_tree(iter, mem) 1293 num++; 1294 return num; 1295} 1296 1297/* 1298 * Return the memory (and swap, if configured) limit for a memcg. 1299 */ 1300u64 mem_cgroup_get_limit(struct mem_cgroup *memcg) 1301{ 1302 u64 limit; 1303 u64 memsw; 1304 1305 limit = res_counter_read_u64(&memcg->res, RES_LIMIT); 1306 limit += total_swap_pages << PAGE_SHIFT; 1307 1308 memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 1309 /* 1310 * If memsw is finite and limits the amount of swap space available 1311 * to this memcg, return that limit. 1312 */ 1313 return min(limit, memsw); 1314} 1315 1316/* 1317 * Visit the first child (need not be the first child as per the ordering 1318 * of the cgroup list, since we track last_scanned_child) of @mem and use 1319 * that to reclaim free pages from. 1320 */ 1321static struct mem_cgroup * 1322mem_cgroup_select_victim(struct mem_cgroup *root_mem) 1323{ 1324 struct mem_cgroup *ret = NULL; 1325 struct cgroup_subsys_state *css; 1326 int nextid, found; 1327 1328 if (!root_mem->use_hierarchy) { 1329 css_get(&root_mem->css); 1330 ret = root_mem; 1331 } 1332 1333 while (!ret) { 1334 rcu_read_lock(); 1335 nextid = root_mem->last_scanned_child + 1; 1336 css = css_get_next(&mem_cgroup_subsys, nextid, &root_mem->css, 1337 &found); 1338 if (css && css_tryget(css)) 1339 ret = container_of(css, struct mem_cgroup, css); 1340 1341 rcu_read_unlock(); 1342 /* Updates scanning parameter */ 1343 spin_lock(&root_mem->reclaim_param_lock); 1344 if (!css) { 1345 /* this means start scan from ID:1 */ 1346 root_mem->last_scanned_child = 0; 1347 } else 1348 root_mem->last_scanned_child = found; 1349 spin_unlock(&root_mem->reclaim_param_lock); 1350 } 1351 1352 return ret; 1353} 1354 1355/* 1356 * Scan the hierarchy if needed to reclaim memory. We remember the last child 1357 * we reclaimed from, so that we don't end up penalizing one child extensively 1358 * based on its position in the children list. 1359 * 1360 * root_mem is the original ancestor that we've been reclaim from. 1361 * 1362 * We give up and return to the caller when we visit root_mem twice. 1363 * (other groups can be removed while we're walking....) 1364 * 1365 * If shrink==true, for avoiding to free too much, this returns immedieately. 1366 */ 1367static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem, 1368 struct zone *zone, 1369 gfp_t gfp_mask, 1370 unsigned long reclaim_options) 1371{ 1372 struct mem_cgroup *victim; 1373 int ret, total = 0; 1374 int loop = 0; 1375 bool noswap = reclaim_options & MEM_CGROUP_RECLAIM_NOSWAP; 1376 bool shrink = reclaim_options & MEM_CGROUP_RECLAIM_SHRINK; 1377 bool check_soft = reclaim_options & MEM_CGROUP_RECLAIM_SOFT; 1378 unsigned long excess = mem_cgroup_get_excess(root_mem); 1379 1380 /* If memsw_is_minimum==1, swap-out is of-no-use. */ 1381 if (root_mem->memsw_is_minimum) 1382 noswap = true; 1383 1384 while (1) { 1385 victim = mem_cgroup_select_victim(root_mem); 1386 if (victim == root_mem) { 1387 loop++; 1388 if (loop >= 1) 1389 drain_all_stock_async(); 1390 if (loop >= 2) { 1391 /* 1392 * If we have not been able to reclaim 1393 * anything, it might because there are 1394 * no reclaimable pages under this hierarchy 1395 */ 1396 if (!check_soft || !total) { 1397 css_put(&victim->css); 1398 break; 1399 } 1400 /* 1401 * We want to do more targetted reclaim. 1402 * excess >> 2 is not to excessive so as to 1403 * reclaim too much, nor too less that we keep 1404 * coming back to reclaim from this cgroup 1405 */ 1406 if (total >= (excess >> 2) || 1407 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) { 1408 css_put(&victim->css); 1409 break; 1410 } 1411 } 1412 } 1413 if (!mem_cgroup_local_usage(victim)) { 1414 /* this cgroup's local usage == 0 */ 1415 css_put(&victim->css); 1416 continue; 1417 } 1418 /* we use swappiness of local cgroup */ 1419 if (check_soft) 1420 ret = mem_cgroup_shrink_node_zone(victim, gfp_mask, 1421 noswap, get_swappiness(victim), zone); 1422 else 1423 ret = try_to_free_mem_cgroup_pages(victim, gfp_mask, 1424 noswap, get_swappiness(victim)); 1425 css_put(&victim->css); 1426 /* 1427 * At shrinking usage, we can't check we should stop here or 1428 * reclaim more. It's depends on callers. last_scanned_child 1429 * will work enough for keeping fairness under tree. 1430 */ 1431 if (shrink) 1432 return ret; 1433 total += ret; 1434 if (check_soft) { 1435 if (res_counter_check_under_soft_limit(&root_mem->res)) 1436 return total; 1437 } else if (mem_cgroup_check_under_limit(root_mem)) 1438 return 1 + total; 1439 } 1440 return total; 1441} 1442 1443/* 1444 * Check OOM-Killer is already running under our hierarchy. 1445 * If someone is running, return false. 1446 */ 1447static bool mem_cgroup_oom_lock(struct mem_cgroup *mem) 1448{ 1449 int x, lock_count = 0; 1450 struct mem_cgroup *iter; 1451 1452 for_each_mem_cgroup_tree(iter, mem) { 1453 x = atomic_inc_return(&iter->oom_lock); 1454 lock_count = max(x, lock_count); 1455 } 1456 1457 if (lock_count == 1) 1458 return true; 1459 return false; 1460} 1461 1462static int mem_cgroup_oom_unlock(struct mem_cgroup *mem) 1463{ 1464 struct mem_cgroup *iter; 1465 1466 /* 1467 * When a new child is created while the hierarchy is under oom, 1468 * mem_cgroup_oom_lock() may not be called. We have to use 1469 * atomic_add_unless() here. 1470 */ 1471 for_each_mem_cgroup_tree(iter, mem) 1472 atomic_add_unless(&iter->oom_lock, -1, 0); 1473 return 0; 1474} 1475 1476 1477static DEFINE_MUTEX(memcg_oom_mutex); 1478static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq); 1479 1480struct oom_wait_info { 1481 struct mem_cgroup *mem; 1482 wait_queue_t wait; 1483}; 1484 1485static int memcg_oom_wake_function(wait_queue_t *wait, 1486 unsigned mode, int sync, void *arg) 1487{ 1488 struct mem_cgroup *wake_mem = (struct mem_cgroup *)arg; 1489 struct oom_wait_info *oom_wait_info; 1490 1491 oom_wait_info = container_of(wait, struct oom_wait_info, wait); 1492 1493 if (oom_wait_info->mem == wake_mem) 1494 goto wakeup; 1495 /* if no hierarchy, no match */ 1496 if (!oom_wait_info->mem->use_hierarchy || !wake_mem->use_hierarchy) 1497 return 0; 1498 /* 1499 * Both of oom_wait_info->mem and wake_mem are stable under us. 1500 * Then we can use css_is_ancestor without taking care of RCU. 1501 */ 1502 if (!css_is_ancestor(&oom_wait_info->mem->css, &wake_mem->css) && 1503 !css_is_ancestor(&wake_mem->css, &oom_wait_info->mem->css)) 1504 return 0; 1505 1506wakeup: 1507 return autoremove_wake_function(wait, mode, sync, arg); 1508} 1509 1510static void memcg_wakeup_oom(struct mem_cgroup *mem) 1511{ 1512 /* for filtering, pass "mem" as argument. */ 1513 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, mem); 1514} 1515 1516static void memcg_oom_recover(struct mem_cgroup *mem) 1517{ 1518 if (mem && atomic_read(&mem->oom_lock)) 1519 memcg_wakeup_oom(mem); 1520} 1521 1522/* 1523 * try to call OOM killer. returns false if we should exit memory-reclaim loop. 1524 */ 1525bool mem_cgroup_handle_oom(struct mem_cgroup *mem, gfp_t mask) 1526{ 1527 struct oom_wait_info owait; 1528 bool locked, need_to_kill; 1529 1530 owait.mem = mem; 1531 owait.wait.flags = 0; 1532 owait.wait.func = memcg_oom_wake_function; 1533 owait.wait.private = current; 1534 INIT_LIST_HEAD(&owait.wait.task_list); 1535 need_to_kill = true; 1536 /* At first, try to OOM lock hierarchy under mem.*/ 1537 mutex_lock(&memcg_oom_mutex); 1538 locked = mem_cgroup_oom_lock(mem); 1539 /* 1540 * Even if signal_pending(), we can't quit charge() loop without 1541 * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL 1542 * under OOM is always welcomed, use TASK_KILLABLE here. 1543 */ 1544 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE); 1545 if (!locked || mem->oom_kill_disable) 1546 need_to_kill = false; 1547 if (locked) 1548 mem_cgroup_oom_notify(mem); 1549 mutex_unlock(&memcg_oom_mutex); 1550 1551 if (need_to_kill) { 1552 finish_wait(&memcg_oom_waitq, &owait.wait); 1553 mem_cgroup_out_of_memory(mem, mask); 1554 } else { 1555 schedule(); 1556 finish_wait(&memcg_oom_waitq, &owait.wait); 1557 } 1558 mutex_lock(&memcg_oom_mutex); 1559 mem_cgroup_oom_unlock(mem); 1560 memcg_wakeup_oom(mem); 1561 mutex_unlock(&memcg_oom_mutex); 1562 1563 if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current)) 1564 return false; 1565 /* Give chance to dying process */ 1566 schedule_timeout(1); 1567 return true; 1568} 1569 1570/* 1571 * Currently used to update mapped file statistics, but the routine can be 1572 * generalized to update other statistics as well. 1573 * 1574 * Notes: Race condition 1575 * 1576 * We usually use page_cgroup_lock() for accessing page_cgroup member but 1577 * it tends to be costly. But considering some conditions, we doesn't need 1578 * to do so _always_. 1579 * 1580 * Considering "charge", lock_page_cgroup() is not required because all 1581 * file-stat operations happen after a page is attached to radix-tree. There 1582 * are no race with "charge". 1583 * 1584 * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup 1585 * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even 1586 * if there are race with "uncharge". Statistics itself is properly handled 1587 * by flags. 1588 * 1589 * Considering "move", this is an only case we see a race. To make the race 1590 * small, we check MEM_CGROUP_ON_MOVE percpu value and detect there are 1591 * possibility of race condition. If there is, we take a lock. 1592 */ 1593 1594void mem_cgroup_update_page_stat(struct page *page, 1595 enum mem_cgroup_page_stat_item idx, int val) 1596{ 1597 struct mem_cgroup *mem; 1598 struct page_cgroup *pc = lookup_page_cgroup(page); 1599 bool need_unlock = false; 1600 unsigned long uninitialized_var(flags); 1601 1602 if (unlikely(!pc)) 1603 return; 1604 1605 rcu_read_lock(); 1606 mem = pc->mem_cgroup; 1607 if (unlikely(!mem || !PageCgroupUsed(pc))) 1608 goto out; 1609 /* pc->mem_cgroup is unstable ? */ 1610 if (unlikely(mem_cgroup_stealed(mem)) || PageTransHuge(page)) { 1611 /* take a lock against to access pc->mem_cgroup */ 1612 move_lock_page_cgroup(pc, &flags); 1613 need_unlock = true; 1614 mem = pc->mem_cgroup; 1615 if (!mem || !PageCgroupUsed(pc)) 1616 goto out; 1617 } 1618 1619 switch (idx) { 1620 case MEMCG_NR_FILE_MAPPED: 1621 if (val > 0) 1622 SetPageCgroupFileMapped(pc); 1623 else if (!page_mapped(page)) 1624 ClearPageCgroupFileMapped(pc); 1625 idx = MEM_CGROUP_STAT_FILE_MAPPED; 1626 break; 1627 default: 1628 BUG(); 1629 } 1630 1631 this_cpu_add(mem->stat->count[idx], val); 1632 1633out: 1634 if (unlikely(need_unlock)) 1635 move_unlock_page_cgroup(pc, &flags); 1636 rcu_read_unlock(); 1637 return; 1638} 1639EXPORT_SYMBOL(mem_cgroup_update_page_stat); 1640 1641/* 1642 * size of first charge trial. "32" comes from vmscan.c's magic value. 1643 * TODO: maybe necessary to use big numbers in big irons. 1644 */ 1645#define CHARGE_SIZE (32 * PAGE_SIZE) 1646struct memcg_stock_pcp { 1647 struct mem_cgroup *cached; /* this never be root cgroup */ 1648 int charge; 1649 struct work_struct work; 1650}; 1651static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock); 1652static atomic_t memcg_drain_count; 1653 1654/* 1655 * Try to consume stocked charge on this cpu. If success, PAGE_SIZE is consumed 1656 * from local stock and true is returned. If the stock is 0 or charges from a 1657 * cgroup which is not current target, returns false. This stock will be 1658 * refilled. 1659 */ 1660static bool consume_stock(struct mem_cgroup *mem) 1661{ 1662 struct memcg_stock_pcp *stock; 1663 bool ret = true; 1664 1665 stock = &get_cpu_var(memcg_stock); 1666 if (mem == stock->cached && stock->charge) 1667 stock->charge -= PAGE_SIZE; 1668 else /* need to call res_counter_charge */ 1669 ret = false; 1670 put_cpu_var(memcg_stock); 1671 return ret; 1672} 1673 1674/* 1675 * Returns stocks cached in percpu to res_counter and reset cached information. 1676 */ 1677static void drain_stock(struct memcg_stock_pcp *stock) 1678{ 1679 struct mem_cgroup *old = stock->cached; 1680 1681 if (stock->charge) { 1682 res_counter_uncharge(&old->res, stock->charge); 1683 if (do_swap_account) 1684 res_counter_uncharge(&old->memsw, stock->charge); 1685 } 1686 stock->cached = NULL; 1687 stock->charge = 0; 1688} 1689 1690/* 1691 * This must be called under preempt disabled or must be called by 1692 * a thread which is pinned to local cpu. 1693 */ 1694static void drain_local_stock(struct work_struct *dummy) 1695{ 1696 struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock); 1697 drain_stock(stock); 1698} 1699 1700/* 1701 * Cache charges(val) which is from res_counter, to local per_cpu area. 1702 * This will be consumed by consume_stock() function, later. 1703 */ 1704static void refill_stock(struct mem_cgroup *mem, int val) 1705{ 1706 struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock); 1707 1708 if (stock->cached != mem) { /* reset if necessary */ 1709 drain_stock(stock); 1710 stock->cached = mem; 1711 } 1712 stock->charge += val; 1713 put_cpu_var(memcg_stock); 1714} 1715 1716/* 1717 * Tries to drain stocked charges in other cpus. This function is asynchronous 1718 * and just put a work per cpu for draining localy on each cpu. Caller can 1719 * expects some charges will be back to res_counter later but cannot wait for 1720 * it. 1721 */ 1722static void drain_all_stock_async(void) 1723{ 1724 int cpu; 1725 /* This function is for scheduling "drain" in asynchronous way. 1726 * The result of "drain" is not directly handled by callers. Then, 1727 * if someone is calling drain, we don't have to call drain more. 1728 * Anyway, WORK_STRUCT_PENDING check in queue_work_on() will catch if 1729 * there is a race. We just do loose check here. 1730 */ 1731 if (atomic_read(&memcg_drain_count)) 1732 return; 1733 /* Notify other cpus that system-wide "drain" is running */ 1734 atomic_inc(&memcg_drain_count); 1735 get_online_cpus(); 1736 for_each_online_cpu(cpu) { 1737 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); 1738 schedule_work_on(cpu, &stock->work); 1739 } 1740 put_online_cpus(); 1741 atomic_dec(&memcg_drain_count); 1742 /* We don't wait for flush_work */ 1743} 1744 1745/* This is a synchronous drain interface. */ 1746static void drain_all_stock_sync(void) 1747{ 1748 /* called when force_empty is called */ 1749 atomic_inc(&memcg_drain_count); 1750 schedule_on_each_cpu(drain_local_stock); 1751 atomic_dec(&memcg_drain_count); 1752} 1753 1754/* 1755 * This function drains percpu counter value from DEAD cpu and 1756 * move it to local cpu. Note that this function can be preempted. 1757 */ 1758static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *mem, int cpu) 1759{ 1760 int i; 1761 1762 spin_lock(&mem->pcp_counter_lock); 1763 for (i = 0; i < MEM_CGROUP_STAT_DATA; i++) { 1764 s64 x = per_cpu(mem->stat->count[i], cpu); 1765 1766 per_cpu(mem->stat->count[i], cpu) = 0; 1767 mem->nocpu_base.count[i] += x; 1768 } 1769 /* need to clear ON_MOVE value, works as a kind of lock. */ 1770 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) = 0; 1771 spin_unlock(&mem->pcp_counter_lock); 1772} 1773 1774static void synchronize_mem_cgroup_on_move(struct mem_cgroup *mem, int cpu) 1775{ 1776 int idx = MEM_CGROUP_ON_MOVE; 1777 1778 spin_lock(&mem->pcp_counter_lock); 1779 per_cpu(mem->stat->count[idx], cpu) = mem->nocpu_base.count[idx]; 1780 spin_unlock(&mem->pcp_counter_lock); 1781} 1782 1783static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb, 1784 unsigned long action, 1785 void *hcpu) 1786{ 1787 int cpu = (unsigned long)hcpu; 1788 struct memcg_stock_pcp *stock; 1789 struct mem_cgroup *iter; 1790 1791 if ((action == CPU_ONLINE)) { 1792 for_each_mem_cgroup_all(iter) 1793 synchronize_mem_cgroup_on_move(iter, cpu); 1794 return NOTIFY_OK; 1795 } 1796 1797 if ((action != CPU_DEAD) || action != CPU_DEAD_FROZEN) 1798 return NOTIFY_OK; 1799 1800 for_each_mem_cgroup_all(iter) 1801 mem_cgroup_drain_pcp_counter(iter, cpu); 1802 1803 stock = &per_cpu(memcg_stock, cpu); 1804 drain_stock(stock); 1805 return NOTIFY_OK; 1806} 1807 1808 1809/* See __mem_cgroup_try_charge() for details */ 1810enum { 1811 CHARGE_OK, /* success */ 1812 CHARGE_RETRY, /* need to retry but retry is not bad */ 1813 CHARGE_NOMEM, /* we can't do more. return -ENOMEM */ 1814 CHARGE_WOULDBLOCK, /* GFP_WAIT wasn't set and no enough res. */ 1815 CHARGE_OOM_DIE, /* the current is killed because of OOM */ 1816}; 1817 1818static int __mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask, 1819 int csize, bool oom_check) 1820{ 1821 struct mem_cgroup *mem_over_limit; 1822 struct res_counter *fail_res; 1823 unsigned long flags = 0; 1824 int ret; 1825 1826 ret = res_counter_charge(&mem->res, csize, &fail_res); 1827 1828 if (likely(!ret)) { 1829 if (!do_swap_account) 1830 return CHARGE_OK; 1831 ret = res_counter_charge(&mem->memsw, csize, &fail_res); 1832 if (likely(!ret)) 1833 return CHARGE_OK; 1834 1835 res_counter_uncharge(&mem->res, csize); 1836 mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw); 1837 flags |= MEM_CGROUP_RECLAIM_NOSWAP; 1838 } else 1839 mem_over_limit = mem_cgroup_from_res_counter(fail_res, res); 1840 1841 if (csize > PAGE_SIZE) /* change csize and retry */ 1842 return CHARGE_RETRY; 1843 1844 if (!(gfp_mask & __GFP_WAIT)) 1845 return CHARGE_WOULDBLOCK; 1846 1847 ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL, 1848 gfp_mask, flags); 1849 /* 1850 * try_to_free_mem_cgroup_pages() might not give us a full 1851 * picture of reclaim. Some pages are reclaimed and might be 1852 * moved to swap cache or just unmapped from the cgroup. 1853 * Check the limit again to see if the reclaim reduced the 1854 * current usage of the cgroup before giving up 1855 */ 1856 if (ret || mem_cgroup_check_under_limit(mem_over_limit)) 1857 return CHARGE_RETRY; 1858 1859 /* 1860 * At task move, charge accounts can be doubly counted. So, it's 1861 * better to wait until the end of task_move if something is going on. 1862 */ 1863 if (mem_cgroup_wait_acct_move(mem_over_limit)) 1864 return CHARGE_RETRY; 1865 1866 /* If we don't need to call oom-killer at el, return immediately */ 1867 if (!oom_check) 1868 return CHARGE_NOMEM; 1869 /* check OOM */ 1870 if (!mem_cgroup_handle_oom(mem_over_limit, gfp_mask)) 1871 return CHARGE_OOM_DIE; 1872 1873 return CHARGE_RETRY; 1874} 1875 1876/* 1877 * Unlike exported interface, "oom" parameter is added. if oom==true, 1878 * oom-killer can be invoked. 1879 */ 1880static int __mem_cgroup_try_charge(struct mm_struct *mm, 1881 gfp_t gfp_mask, 1882 struct mem_cgroup **memcg, bool oom, 1883 int page_size) 1884{ 1885 int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES; 1886 struct mem_cgroup *mem = NULL; 1887 int ret; 1888 int csize = max(CHARGE_SIZE, (unsigned long) page_size); 1889 1890 /* 1891 * Unlike gloval-vm's OOM-kill, we're not in memory shortage 1892 * in system level. So, allow to go ahead dying process in addition to 1893 * MEMDIE process. 1894 */ 1895 if (unlikely(test_thread_flag(TIF_MEMDIE) 1896 || fatal_signal_pending(current))) 1897 goto bypass; 1898 1899 /* 1900 * We always charge the cgroup the mm_struct belongs to. 1901 * The mm_struct's mem_cgroup changes on task migration if the 1902 * thread group leader migrates. It's possible that mm is not 1903 * set, if so charge the init_mm (happens for pagecache usage). 1904 */ 1905 if (!*memcg && !mm) 1906 goto bypass; 1907again: 1908 if (*memcg) { /* css should be a valid one */ 1909 mem = *memcg; 1910 VM_BUG_ON(css_is_removed(&mem->css)); 1911 if (mem_cgroup_is_root(mem)) 1912 goto done; 1913 if (page_size == PAGE_SIZE && consume_stock(mem)) 1914 goto done; 1915 css_get(&mem->css); 1916 } else { 1917 struct task_struct *p; 1918 1919 rcu_read_lock(); 1920 p = rcu_dereference(mm->owner); 1921 /* 1922 * Because we don't have task_lock(), "p" can exit. 1923 * In that case, "mem" can point to root or p can be NULL with 1924 * race with swapoff. Then, we have small risk of mis-accouning. 1925 * But such kind of mis-account by race always happens because 1926 * we don't have cgroup_mutex(). It's overkill and we allo that 1927 * small race, here. 1928 * (*) swapoff at el will charge against mm-struct not against 1929 * task-struct. So, mm->owner can be NULL. 1930 */ 1931 mem = mem_cgroup_from_task(p); 1932 if (!mem || mem_cgroup_is_root(mem)) { 1933 rcu_read_unlock(); 1934 goto done; 1935 } 1936 if (page_size == PAGE_SIZE && consume_stock(mem)) { 1937 /* 1938 * It seems dagerous to access memcg without css_get(). 1939 * But considering how consume_stok works, it's not 1940 * necessary. If consume_stock success, some charges 1941 * from this memcg are cached on this cpu. So, we 1942 * don't need to call css_get()/css_tryget() before 1943 * calling consume_stock(). 1944 */ 1945 rcu_read_unlock(); 1946 goto done; 1947 } 1948 /* after here, we may be blocked. we need to get refcnt */ 1949 if (!css_tryget(&mem->css)) { 1950 rcu_read_unlock(); 1951 goto again; 1952 } 1953 rcu_read_unlock(); 1954 } 1955 1956 do { 1957 bool oom_check; 1958 1959 /* If killed, bypass charge */ 1960 if (fatal_signal_pending(current)) { 1961 css_put(&mem->css); 1962 goto bypass; 1963 } 1964 1965 oom_check = false; 1966 if (oom && !nr_oom_retries) { 1967 oom_check = true; 1968 nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES; 1969 } 1970 1971 ret = __mem_cgroup_do_charge(mem, gfp_mask, csize, oom_check); 1972 1973 switch (ret) { 1974 case CHARGE_OK: 1975 break; 1976 case CHARGE_RETRY: /* not in OOM situation but retry */ 1977 csize = page_size; 1978 css_put(&mem->css); 1979 mem = NULL; 1980 goto again; 1981 case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */ 1982 css_put(&mem->css); 1983 goto nomem; 1984 case CHARGE_NOMEM: /* OOM routine works */ 1985 if (!oom) { 1986 css_put(&mem->css); 1987 goto nomem; 1988 } 1989 /* If oom, we never return -ENOMEM */ 1990 nr_oom_retries--; 1991 break; 1992 case CHARGE_OOM_DIE: /* Killed by OOM Killer */ 1993 css_put(&mem->css); 1994 goto bypass; 1995 } 1996 } while (ret != CHARGE_OK); 1997 1998 if (csize > page_size) 1999 refill_stock(mem, csize - page_size); 2000 css_put(&mem->css); 2001done: 2002 *memcg = mem; 2003 return 0; 2004nomem: 2005 *memcg = NULL; 2006 return -ENOMEM; 2007bypass: 2008 *memcg = NULL; 2009 return 0; 2010} 2011 2012/* 2013 * Somemtimes we have to undo a charge we got by try_charge(). 2014 * This function is for that and do uncharge, put css's refcnt. 2015 * gotten by try_charge(). 2016 */ 2017static void __mem_cgroup_cancel_charge(struct mem_cgroup *mem, 2018 unsigned long count) 2019{ 2020 if (!mem_cgroup_is_root(mem)) { 2021 res_counter_uncharge(&mem->res, PAGE_SIZE * count); 2022 if (do_swap_account) 2023 res_counter_uncharge(&mem->memsw, PAGE_SIZE * count); 2024 } 2025} 2026 2027static void mem_cgroup_cancel_charge(struct mem_cgroup *mem, 2028 int page_size) 2029{ 2030 __mem_cgroup_cancel_charge(mem, page_size >> PAGE_SHIFT); 2031} 2032 2033/* 2034 * A helper function to get mem_cgroup from ID. must be called under 2035 * rcu_read_lock(). The caller must check css_is_removed() or some if 2036 * it's concern. (dropping refcnt from swap can be called against removed 2037 * memcg.) 2038 */ 2039static struct mem_cgroup *mem_cgroup_lookup(unsigned short id) 2040{ 2041 struct cgroup_subsys_state *css; 2042 2043 /* ID 0 is unused ID */ 2044 if (!id) 2045 return NULL; 2046 css = css_lookup(&mem_cgroup_subsys, id); 2047 if (!css) 2048 return NULL; 2049 return container_of(css, struct mem_cgroup, css); 2050} 2051 2052struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page) 2053{ 2054 struct mem_cgroup *mem = NULL; 2055 struct page_cgroup *pc; 2056 unsigned short id; 2057 swp_entry_t ent; 2058 2059 VM_BUG_ON(!PageLocked(page)); 2060 2061 pc = lookup_page_cgroup(page); 2062 lock_page_cgroup(pc); 2063 if (PageCgroupUsed(pc)) { 2064 mem = pc->mem_cgroup; 2065 if (mem && !css_tryget(&mem->css)) 2066 mem = NULL; 2067 } else if (PageSwapCache(page)) { 2068 ent.val = page_private(page); 2069 id = lookup_swap_cgroup(ent); 2070 rcu_read_lock(); 2071 mem = mem_cgroup_lookup(id); 2072 if (mem && !css_tryget(&mem->css)) 2073 mem = NULL; 2074 rcu_read_unlock(); 2075 } 2076 unlock_page_cgroup(pc); 2077 return mem; 2078} 2079 2080static void __mem_cgroup_commit_charge(struct mem_cgroup *mem, 2081 struct page_cgroup *pc, 2082 enum charge_type ctype, 2083 int page_size) 2084{ 2085 int nr_pages = page_size >> PAGE_SHIFT; 2086 2087 /* try_charge() can return NULL to *memcg, taking care of it. */ 2088 if (!mem) 2089 return; 2090 2091 lock_page_cgroup(pc); 2092 if (unlikely(PageCgroupUsed(pc))) { 2093 unlock_page_cgroup(pc); 2094 mem_cgroup_cancel_charge(mem, page_size); 2095 return; 2096 } 2097 /* 2098 * we don't need page_cgroup_lock about tail pages, becase they are not 2099 * accessed by any other context at this point. 2100 */ 2101 pc->mem_cgroup = mem; 2102 /* 2103 * We access a page_cgroup asynchronously without lock_page_cgroup(). 2104 * Especially when a page_cgroup is taken from a page, pc->mem_cgroup 2105 * is accessed after testing USED bit. To make pc->mem_cgroup visible 2106 * before USED bit, we need memory barrier here. 2107 * See mem_cgroup_add_lru_list(), etc. 2108 */ 2109 smp_wmb(); 2110 switch (ctype) { 2111 case MEM_CGROUP_CHARGE_TYPE_CACHE: 2112 case MEM_CGROUP_CHARGE_TYPE_SHMEM: 2113 SetPageCgroupCache(pc); 2114 SetPageCgroupUsed(pc); 2115 break; 2116 case MEM_CGROUP_CHARGE_TYPE_MAPPED: 2117 ClearPageCgroupCache(pc); 2118 SetPageCgroupUsed(pc); 2119 break; 2120 default: 2121 break; 2122 } 2123 2124 mem_cgroup_charge_statistics(mem, PageCgroupCache(pc), nr_pages); 2125 unlock_page_cgroup(pc); 2126 /* 2127 * "charge_statistics" updated event counter. Then, check it. 2128 * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree. 2129 * if they exceeds softlimit. 2130 */ 2131 memcg_check_events(mem, pc->page); 2132} 2133 2134#ifdef CONFIG_TRANSPARENT_HUGEPAGE 2135 2136#define PCGF_NOCOPY_AT_SPLIT ((1 << PCG_LOCK) | (1 << PCG_MOVE_LOCK) |\ 2137 (1 << PCG_ACCT_LRU) | (1 << PCG_MIGRATION)) 2138/* 2139 * Because tail pages are not marked as "used", set it. We're under 2140 * zone->lru_lock, 'splitting on pmd' and compund_lock. 2141 */ 2142void mem_cgroup_split_huge_fixup(struct page *head, struct page *tail) 2143{ 2144 struct page_cgroup *head_pc = lookup_page_cgroup(head); 2145 struct page_cgroup *tail_pc = lookup_page_cgroup(tail); 2146 unsigned long flags; 2147 2148 if (mem_cgroup_disabled()) 2149 return; 2150 /* 2151 * We have no races with charge/uncharge but will have races with 2152 * page state accounting. 2153 */ 2154 move_lock_page_cgroup(head_pc, &flags); 2155 2156 tail_pc->mem_cgroup = head_pc->mem_cgroup; 2157 smp_wmb(); /* see __commit_charge() */ 2158 if (PageCgroupAcctLRU(head_pc)) { 2159 enum lru_list lru; 2160 struct mem_cgroup_per_zone *mz; 2161 2162 /* 2163 * LRU flags cannot be copied because we need to add tail 2164 *.page to LRU by generic call and our hook will be called. 2165 * We hold lru_lock, then, reduce counter directly. 2166 */ 2167 lru = page_lru(head); 2168 mz = page_cgroup_zoneinfo(head_pc); 2169 MEM_CGROUP_ZSTAT(mz, lru) -= 1; 2170 } 2171 tail_pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT; 2172 move_unlock_page_cgroup(head_pc, &flags); 2173} 2174#endif 2175 2176/** 2177 * __mem_cgroup_move_account - move account of the page 2178 * @pc: page_cgroup of the page. 2179 * @from: mem_cgroup which the page is moved from. 2180 * @to: mem_cgroup which the page is moved to. @from != @to. 2181 * @uncharge: whether we should call uncharge and css_put against @from. 2182 * 2183 * The caller must confirm following. 2184 * - page is not on LRU (isolate_page() is useful.) 2185 * - the pc is locked, used, and ->mem_cgroup points to @from. 2186 * 2187 * This function doesn't do "charge" nor css_get to new cgroup. It should be 2188 * done by a caller(__mem_cgroup_try_charge would be usefull). If @uncharge is 2189 * true, this function does "uncharge" from old cgroup, but it doesn't if 2190 * @uncharge is false, so a caller should do "uncharge". 2191 */ 2192 2193static void __mem_cgroup_move_account(struct page_cgroup *pc, 2194 struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge, 2195 int charge_size) 2196{ 2197 int nr_pages = charge_size >> PAGE_SHIFT; 2198 2199 VM_BUG_ON(from == to); 2200 VM_BUG_ON(PageLRU(pc->page)); 2201 VM_BUG_ON(!page_is_cgroup_locked(pc)); 2202 VM_BUG_ON(!PageCgroupUsed(pc)); 2203 VM_BUG_ON(pc->mem_cgroup != from); 2204 2205 if (PageCgroupFileMapped(pc)) { 2206 /* Update mapped_file data for mem_cgroup */ 2207 preempt_disable(); 2208 __this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]); 2209 __this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]); 2210 preempt_enable(); 2211 } 2212 mem_cgroup_charge_statistics(from, PageCgroupCache(pc), -nr_pages); 2213 if (uncharge) 2214 /* This is not "cancel", but cancel_charge does all we need. */ 2215 mem_cgroup_cancel_charge(from, charge_size); 2216 2217 /* caller should have done css_get */ 2218 pc->mem_cgroup = to; 2219 mem_cgroup_charge_statistics(to, PageCgroupCache(pc), nr_pages); 2220 /* 2221 * We charges against "to" which may not have any tasks. Then, "to" 2222 * can be under rmdir(). But in current implementation, caller of 2223 * this function is just force_empty() and move charge, so it's 2224 * garanteed that "to" is never removed. So, we don't check rmdir 2225 * status here. 2226 */ 2227} 2228 2229/* 2230 * check whether the @pc is valid for moving account and call 2231 * __mem_cgroup_move_account() 2232 */ 2233static int mem_cgroup_move_account(struct page_cgroup *pc, 2234 struct mem_cgroup *from, struct mem_cgroup *to, 2235 bool uncharge, int charge_size) 2236{ 2237 int ret = -EINVAL; 2238 unsigned long flags; 2239 /* 2240 * The page is isolated from LRU. So, collapse function 2241 * will not handle this page. But page splitting can happen. 2242 * Do this check under compound_page_lock(). The caller should 2243 * hold it. 2244 */ 2245 if ((charge_size > PAGE_SIZE) && !PageTransHuge(pc->page)) 2246 return -EBUSY; 2247 2248 lock_page_cgroup(pc); 2249 if (PageCgroupUsed(pc) && pc->mem_cgroup == from) { 2250 move_lock_page_cgroup(pc, &flags); 2251 __mem_cgroup_move_account(pc, from, to, uncharge, charge_size); 2252 move_unlock_page_cgroup(pc, &flags); 2253 ret = 0; 2254 } 2255 unlock_page_cgroup(pc); 2256 /* 2257 * check events 2258 */ 2259 memcg_check_events(to, pc->page); 2260 memcg_check_events(from, pc->page); 2261 return ret; 2262} 2263 2264/* 2265 * move charges to its parent. 2266 */ 2267 2268static int mem_cgroup_move_parent(struct page_cgroup *pc, 2269 struct mem_cgroup *child, 2270 gfp_t gfp_mask) 2271{ 2272 struct page *page = pc->page; 2273 struct cgroup *cg = child->css.cgroup; 2274 struct cgroup *pcg = cg->parent; 2275 struct mem_cgroup *parent; 2276 int page_size = PAGE_SIZE; 2277 unsigned long flags; 2278 int ret; 2279 2280 /* Is ROOT ? */ 2281 if (!pcg) 2282 return -EINVAL; 2283 2284 ret = -EBUSY; 2285 if (!get_page_unless_zero(page)) 2286 goto out; 2287 if (isolate_lru_page(page)) 2288 goto put; 2289 2290 if (PageTransHuge(page)) 2291 page_size = HPAGE_SIZE; 2292 2293 parent = mem_cgroup_from_cont(pcg); 2294 ret = __mem_cgroup_try_charge(NULL, gfp_mask, 2295 &parent, false, page_size); 2296 if (ret || !parent) 2297 goto put_back; 2298 2299 if (page_size > PAGE_SIZE) 2300 flags = compound_lock_irqsave(page); 2301 2302 ret = mem_cgroup_move_account(pc, child, parent, true, page_size); 2303 if (ret) 2304 mem_cgroup_cancel_charge(parent, page_size); 2305 2306 if (page_size > PAGE_SIZE) 2307 compound_unlock_irqrestore(page, flags); 2308put_back: 2309 putback_lru_page(page); 2310put: 2311 put_page(page); 2312out: 2313 return ret; 2314} 2315 2316/* 2317 * Charge the memory controller for page usage. 2318 * Return 2319 * 0 if the charge was successful 2320 * < 0 if the cgroup is over its limit 2321 */ 2322static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm, 2323 gfp_t gfp_mask, enum charge_type ctype) 2324{ 2325 struct mem_cgroup *mem = NULL; 2326 struct page_cgroup *pc; 2327 int ret; 2328 int page_size = PAGE_SIZE; 2329 2330 if (PageTransHuge(page)) { 2331 page_size <<= compound_order(page); 2332 VM_BUG_ON(!PageTransHuge(page)); 2333 } 2334 2335 pc = lookup_page_cgroup(page); 2336 /* can happen at boot */ 2337 if (unlikely(!pc)) 2338 return 0; 2339 prefetchw(pc); 2340 2341 ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, true, page_size); 2342 if (ret || !mem) 2343 return ret; 2344 2345 __mem_cgroup_commit_charge(mem, pc, ctype, page_size); 2346 return 0; 2347} 2348 2349int mem_cgroup_newpage_charge(struct page *page, 2350 struct mm_struct *mm, gfp_t gfp_mask) 2351{ 2352 if (mem_cgroup_disabled()) 2353 return 0; 2354 /* 2355 * If already mapped, we don't have to account. 2356 * If page cache, page->mapping has address_space. 2357 * But page->mapping may have out-of-use anon_vma pointer, 2358 * detecit it by PageAnon() check. newly-mapped-anon's page->mapping 2359 * is NULL. 2360 */ 2361 if (page_mapped(page) || (page->mapping && !PageAnon(page))) 2362 return 0; 2363 if (unlikely(!mm)) 2364 mm = &init_mm; 2365 return mem_cgroup_charge_common(page, mm, gfp_mask, 2366 MEM_CGROUP_CHARGE_TYPE_MAPPED); 2367} 2368 2369static void 2370__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr, 2371 enum charge_type ctype); 2372 2373int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm, 2374 gfp_t gfp_mask) 2375{ 2376 int ret; 2377 2378 if (mem_cgroup_disabled()) 2379 return 0; 2380 if (PageCompound(page)) 2381 return 0; 2382 /* 2383 * Corner case handling. This is called from add_to_page_cache() 2384 * in usual. But some FS (shmem) precharges this page before calling it 2385 * and call add_to_page_cache() with GFP_NOWAIT. 2386 * 2387 * For GFP_NOWAIT case, the page may be pre-charged before calling 2388 * add_to_page_cache(). (See shmem.c) check it here and avoid to call 2389 * charge twice. (It works but has to pay a bit larger cost.) 2390 * And when the page is SwapCache, it should take swap information 2391 * into account. This is under lock_page() now. 2392 */ 2393 if (!(gfp_mask & __GFP_WAIT)) { 2394 struct page_cgroup *pc; 2395 2396 pc = lookup_page_cgroup(page); 2397 if (!pc) 2398 return 0; 2399 lock_page_cgroup(pc); 2400 if (PageCgroupUsed(pc)) { 2401 unlock_page_cgroup(pc); 2402 return 0; 2403 } 2404 unlock_page_cgroup(pc); 2405 } 2406 2407 if (unlikely(!mm)) 2408 mm = &init_mm; 2409 2410 if (page_is_file_cache(page)) 2411 return mem_cgroup_charge_common(page, mm, gfp_mask, 2412 MEM_CGROUP_CHARGE_TYPE_CACHE); 2413 2414 /* shmem */ 2415 if (PageSwapCache(page)) { 2416 struct mem_cgroup *mem = NULL; 2417 2418 ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem); 2419 if (!ret) 2420 __mem_cgroup_commit_charge_swapin(page, mem, 2421 MEM_CGROUP_CHARGE_TYPE_SHMEM); 2422 } else 2423 ret = mem_cgroup_charge_common(page, mm, gfp_mask, 2424 MEM_CGROUP_CHARGE_TYPE_SHMEM); 2425 2426 return ret; 2427} 2428 2429/* 2430 * While swap-in, try_charge -> commit or cancel, the page is locked. 2431 * And when try_charge() successfully returns, one refcnt to memcg without 2432 * struct page_cgroup is acquired. This refcnt will be consumed by 2433 * "commit()" or removed by "cancel()" 2434 */ 2435int mem_cgroup_try_charge_swapin(struct mm_struct *mm, 2436 struct page *page, 2437 gfp_t mask, struct mem_cgroup **ptr) 2438{ 2439 struct mem_cgroup *mem; 2440 int ret; 2441 2442 if (mem_cgroup_disabled()) 2443 return 0; 2444 2445 if (!do_swap_account) 2446 goto charge_cur_mm; 2447 /* 2448 * A racing thread's fault, or swapoff, may have already updated 2449 * the pte, and even removed page from swap cache: in those cases 2450 * do_swap_page()'s pte_same() test will fail; but there's also a 2451 * KSM case which does need to charge the page. 2452 */ 2453 if (!PageSwapCache(page)) 2454 goto charge_cur_mm; 2455 mem = try_get_mem_cgroup_from_page(page); 2456 if (!mem) 2457 goto charge_cur_mm; 2458 *ptr = mem; 2459 ret = __mem_cgroup_try_charge(NULL, mask, ptr, true, PAGE_SIZE); 2460 css_put(&mem->css); 2461 return ret; 2462charge_cur_mm: 2463 if (unlikely(!mm)) 2464 mm = &init_mm; 2465 return __mem_cgroup_try_charge(mm, mask, ptr, true, PAGE_SIZE); 2466} 2467 2468static void 2469__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr, 2470 enum charge_type ctype) 2471{ 2472 struct page_cgroup *pc; 2473 2474 if (mem_cgroup_disabled()) 2475 return; 2476 if (!ptr) 2477 return; 2478 cgroup_exclude_rmdir(&ptr->css); 2479 pc = lookup_page_cgroup(page); 2480 mem_cgroup_lru_del_before_commit_swapcache(page); 2481 __mem_cgroup_commit_charge(ptr, pc, ctype, PAGE_SIZE); 2482 mem_cgroup_lru_add_after_commit_swapcache(page); 2483 /* 2484 * Now swap is on-memory. This means this page may be 2485 * counted both as mem and swap....double count. 2486 * Fix it by uncharging from memsw. Basically, this SwapCache is stable 2487 * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page() 2488 * may call delete_from_swap_cache() before reach here. 2489 */ 2490 if (do_swap_account && PageSwapCache(page)) { 2491 swp_entry_t ent = {.val = page_private(page)}; 2492 unsigned short id; 2493 struct mem_cgroup *memcg; 2494 2495 id = swap_cgroup_record(ent, 0); 2496 rcu_read_lock(); 2497 memcg = mem_cgroup_lookup(id); 2498 if (memcg) { 2499 /* 2500 * This recorded memcg can be obsolete one. So, avoid 2501 * calling css_tryget 2502 */ 2503 if (!mem_cgroup_is_root(memcg)) 2504 res_counter_uncharge(&memcg->memsw, PAGE_SIZE); 2505 mem_cgroup_swap_statistics(memcg, false); 2506 mem_cgroup_put(memcg); 2507 } 2508 rcu_read_unlock(); 2509 } 2510 /* 2511 * At swapin, we may charge account against cgroup which has no tasks. 2512 * So, rmdir()->pre_destroy() can be called while we do this charge. 2513 * In that case, we need to call pre_destroy() again. check it here. 2514 */ 2515 cgroup_release_and_wakeup_rmdir(&ptr->css); 2516} 2517 2518void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr) 2519{ 2520 __mem_cgroup_commit_charge_swapin(page, ptr, 2521 MEM_CGROUP_CHARGE_TYPE_MAPPED); 2522} 2523 2524void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem) 2525{ 2526 if (mem_cgroup_disabled()) 2527 return; 2528 if (!mem) 2529 return; 2530 mem_cgroup_cancel_charge(mem, PAGE_SIZE); 2531} 2532 2533static void 2534__do_uncharge(struct mem_cgroup *mem, const enum charge_type ctype, 2535 int page_size) 2536{ 2537 struct memcg_batch_info *batch = NULL; 2538 bool uncharge_memsw = true; 2539 /* If swapout, usage of swap doesn't decrease */ 2540 if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) 2541 uncharge_memsw = false; 2542 2543 batch = ¤t->memcg_batch; 2544 /* 2545 * In usual, we do css_get() when we remember memcg pointer. 2546 * But in this case, we keep res->usage until end of a series of 2547 * uncharges. Then, it's ok to ignore memcg's refcnt. 2548 */ 2549 if (!batch->memcg) 2550 batch->memcg = mem; 2551 /* 2552 * do_batch > 0 when unmapping pages or inode invalidate/truncate. 2553 * In those cases, all pages freed continously can be expected to be in 2554 * the same cgroup and we have chance to coalesce uncharges. 2555 * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE) 2556 * because we want to do uncharge as soon as possible. 2557 */ 2558 2559 if (!batch->do_batch || test_thread_flag(TIF_MEMDIE)) 2560 goto direct_uncharge; 2561 2562 if (page_size != PAGE_SIZE) 2563 goto direct_uncharge; 2564 2565 /* 2566 * In typical case, batch->memcg == mem. This means we can 2567 * merge a series of uncharges to an uncharge of res_counter. 2568 * If not, we uncharge res_counter ony by one. 2569 */ 2570 if (batch->memcg != mem) 2571 goto direct_uncharge; 2572 /* remember freed charge and uncharge it later */ 2573 batch->bytes += PAGE_SIZE; 2574 if (uncharge_memsw) 2575 batch->memsw_bytes += PAGE_SIZE; 2576 return; 2577direct_uncharge: 2578 res_counter_uncharge(&mem->res, page_size); 2579 if (uncharge_memsw) 2580 res_counter_uncharge(&mem->memsw, page_size); 2581 if (unlikely(batch->memcg != mem)) 2582 memcg_oom_recover(mem); 2583 return; 2584} 2585 2586/* 2587 * uncharge if !page_mapped(page) 2588 */ 2589static struct mem_cgroup * 2590__mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype) 2591{ 2592 int count; 2593 struct page_cgroup *pc; 2594 struct mem_cgroup *mem = NULL; 2595 int page_size = PAGE_SIZE; 2596 2597 if (mem_cgroup_disabled()) 2598 return NULL; 2599 2600 if (PageSwapCache(page)) 2601 return NULL; 2602 2603 if (PageTransHuge(page)) { 2604 page_size <<= compound_order(page); 2605 VM_BUG_ON(!PageTransHuge(page)); 2606 } 2607 2608 count = page_size >> PAGE_SHIFT; 2609 /* 2610 * Check if our page_cgroup is valid 2611 */ 2612 pc = lookup_page_cgroup(page); 2613 if (unlikely(!pc || !PageCgroupUsed(pc))) 2614 return NULL; 2615 2616 lock_page_cgroup(pc); 2617 2618 mem = pc->mem_cgroup; 2619 2620 if (!PageCgroupUsed(pc)) 2621 goto unlock_out; 2622 2623 switch (ctype) { 2624 case MEM_CGROUP_CHARGE_TYPE_MAPPED: 2625 case MEM_CGROUP_CHARGE_TYPE_DROP: 2626 /* See mem_cgroup_prepare_migration() */ 2627 if (page_mapped(page) || PageCgroupMigration(pc)) 2628 goto unlock_out; 2629 break; 2630 case MEM_CGROUP_CHARGE_TYPE_SWAPOUT: 2631 if (!PageAnon(page)) { /* Shared memory */ 2632 if (page->mapping && !page_is_file_cache(page)) 2633 goto unlock_out; 2634 } else if (page_mapped(page)) /* Anon */ 2635 goto unlock_out; 2636 break; 2637 default: 2638 break; 2639 } 2640 2641 mem_cgroup_charge_statistics(mem, PageCgroupCache(pc), -count); 2642 2643 ClearPageCgroupUsed(pc); 2644 /* 2645 * pc->mem_cgroup is not cleared here. It will be accessed when it's 2646 * freed from LRU. This is safe because uncharged page is expected not 2647 * to be reused (freed soon). Exception is SwapCache, it's handled by 2648 * special functions. 2649 */ 2650 2651 unlock_page_cgroup(pc); 2652 /* 2653 * even after unlock, we have mem->res.usage here and this memcg 2654 * will never be freed. 2655 */ 2656 memcg_check_events(mem, page); 2657 if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) { 2658 mem_cgroup_swap_statistics(mem, true); 2659 mem_cgroup_get(mem); 2660 } 2661 if (!mem_cgroup_is_root(mem)) 2662 __do_uncharge(mem, ctype, page_size); 2663 2664 return mem; 2665 2666unlock_out: 2667 unlock_page_cgroup(pc); 2668 return NULL; 2669} 2670 2671void mem_cgroup_uncharge_page(struct page *page) 2672{ 2673 /* early check. */ 2674 if (page_mapped(page)) 2675 return; 2676 if (page->mapping && !PageAnon(page)) 2677 return; 2678 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED); 2679} 2680 2681void mem_cgroup_uncharge_cache_page(struct page *page) 2682{ 2683 VM_BUG_ON(page_mapped(page)); 2684 VM_BUG_ON(page->mapping); 2685 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE); 2686} 2687 2688/* 2689 * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate. 2690 * In that cases, pages are freed continuously and we can expect pages 2691 * are in the same memcg. All these calls itself limits the number of 2692 * pages freed at once, then uncharge_start/end() is called properly. 2693 * This may be called prural(2) times in a context, 2694 */ 2695 2696void mem_cgroup_uncharge_start(void) 2697{ 2698 current->memcg_batch.do_batch++; 2699 /* We can do nest. */ 2700 if (current->memcg_batch.do_batch == 1) { 2701 current->memcg_batch.memcg = NULL; 2702 current->memcg_batch.bytes = 0; 2703 current->memcg_batch.memsw_bytes = 0; 2704 } 2705} 2706 2707void mem_cgroup_uncharge_end(void) 2708{ 2709 struct memcg_batch_info *batch = ¤t->memcg_batch; 2710 2711 if (!batch->do_batch) 2712 return; 2713 2714 batch->do_batch--; 2715 if (batch->do_batch) /* If stacked, do nothing. */ 2716 return; 2717 2718 if (!batch->memcg) 2719 return; 2720 /* 2721 * This "batch->memcg" is valid without any css_get/put etc... 2722 * bacause we hide charges behind us. 2723 */ 2724 if (batch->bytes) 2725 res_counter_uncharge(&batch->memcg->res, batch->bytes); 2726 if (batch->memsw_bytes) 2727 res_counter_uncharge(&batch->memcg->memsw, batch->memsw_bytes); 2728 memcg_oom_recover(batch->memcg); 2729 /* forget this pointer (for sanity check) */ 2730 batch->memcg = NULL; 2731} 2732 2733#ifdef CONFIG_SWAP 2734/* 2735 * called after __delete_from_swap_cache() and drop "page" account. 2736 * memcg information is recorded to swap_cgroup of "ent" 2737 */ 2738void 2739mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout) 2740{ 2741 struct mem_cgroup *memcg; 2742 int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT; 2743 2744 if (!swapout) /* this was a swap cache but the swap is unused ! */ 2745 ctype = MEM_CGROUP_CHARGE_TYPE_DROP; 2746 2747 memcg = __mem_cgroup_uncharge_common(page, ctype); 2748 2749 /* 2750 * record memcg information, if swapout && memcg != NULL, 2751 * mem_cgroup_get() was called in uncharge(). 2752 */ 2753 if (do_swap_account && swapout && memcg) 2754 swap_cgroup_record(ent, css_id(&memcg->css)); 2755} 2756#endif 2757 2758#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP 2759/* 2760 * called from swap_entry_free(). remove record in swap_cgroup and 2761 * uncharge "memsw" account. 2762 */ 2763void mem_cgroup_uncharge_swap(swp_entry_t ent) 2764{ 2765 struct mem_cgroup *memcg; 2766 unsigned short id; 2767 2768 if (!do_swap_account) 2769 return; 2770 2771 id = swap_cgroup_record(ent, 0); 2772 rcu_read_lock(); 2773 memcg = mem_cgroup_lookup(id); 2774 if (memcg) { 2775 /* 2776 * We uncharge this because swap is freed. 2777 * This memcg can be obsolete one. We avoid calling css_tryget 2778 */ 2779 if (!mem_cgroup_is_root(memcg)) 2780 res_counter_uncharge(&memcg->memsw, PAGE_SIZE); 2781 mem_cgroup_swap_statistics(memcg, false); 2782 mem_cgroup_put(memcg); 2783 } 2784 rcu_read_unlock(); 2785} 2786 2787/** 2788 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record. 2789 * @entry: swap entry to be moved 2790 * @from: mem_cgroup which the entry is moved from 2791 * @to: mem_cgroup which the entry is moved to 2792 * @need_fixup: whether we should fixup res_counters and refcounts. 2793 * 2794 * It succeeds only when the swap_cgroup's record for this entry is the same 2795 * as the mem_cgroup's id of @from. 2796 * 2797 * Returns 0 on success, -EINVAL on failure. 2798 * 2799 * The caller must have charged to @to, IOW, called res_counter_charge() about 2800 * both res and memsw, and called css_get(). 2801 */ 2802static int mem_cgroup_move_swap_account(swp_entry_t entry, 2803 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup) 2804{ 2805 unsigned short old_id, new_id; 2806 2807 old_id = css_id(&from->css); 2808 new_id = css_id(&to->css); 2809 2810 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) { 2811 mem_cgroup_swap_statistics(from, false); 2812 mem_cgroup_swap_statistics(to, true); 2813 /* 2814 * This function is only called from task migration context now. 2815 * It postpones res_counter and refcount handling till the end 2816 * of task migration(mem_cgroup_clear_mc()) for performance 2817 * improvement. But we cannot postpone mem_cgroup_get(to) 2818 * because if the process that has been moved to @to does 2819 * swap-in, the refcount of @to might be decreased to 0. 2820 */ 2821 mem_cgroup_get(to); 2822 if (need_fixup) { 2823 if (!mem_cgroup_is_root(from)) 2824 res_counter_uncharge(&from->memsw, PAGE_SIZE); 2825 mem_cgroup_put(from); 2826 /* 2827 * we charged both to->res and to->memsw, so we should 2828 * uncharge to->res. 2829 */ 2830 if (!mem_cgroup_is_root(to)) 2831 res_counter_uncharge(&to->res, PAGE_SIZE); 2832 } 2833 return 0; 2834 } 2835 return -EINVAL; 2836} 2837#else 2838static inline int mem_cgroup_move_swap_account(swp_entry_t entry, 2839 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup) 2840{ 2841 return -EINVAL; 2842} 2843#endif 2844 2845/* 2846 * Before starting migration, account PAGE_SIZE to mem_cgroup that the old 2847 * page belongs to. 2848 */ 2849int mem_cgroup_prepare_migration(struct page *page, 2850 struct page *newpage, struct mem_cgroup **ptr) 2851{ 2852 struct page_cgroup *pc; 2853 struct mem_cgroup *mem = NULL; 2854 enum charge_type ctype; 2855 int ret = 0; 2856 2857 VM_BUG_ON(PageTransHuge(page)); 2858 if (mem_cgroup_disabled()) 2859 return 0; 2860 2861 pc = lookup_page_cgroup(page); 2862 lock_page_cgroup(pc); 2863 if (PageCgroupUsed(pc)) { 2864 mem = pc->mem_cgroup; 2865 css_get(&mem->css); 2866 /* 2867 * At migrating an anonymous page, its mapcount goes down 2868 * to 0 and uncharge() will be called. But, even if it's fully 2869 * unmapped, migration may fail and this page has to be 2870 * charged again. We set MIGRATION flag here and delay uncharge 2871 * until end_migration() is called 2872 * 2873 * Corner Case Thinking 2874 * A) 2875 * When the old page was mapped as Anon and it's unmap-and-freed 2876 * while migration was ongoing. 2877 * If unmap finds the old page, uncharge() of it will be delayed 2878 * until end_migration(). If unmap finds a new page, it's 2879 * uncharged when it make mapcount to be 1->0. If unmap code 2880 * finds swap_migration_entry, the new page will not be mapped 2881 * and end_migration() will find it(mapcount==0). 2882 * 2883 * B) 2884 * When the old page was mapped but migraion fails, the kernel 2885 * remaps it. A charge for it is kept by MIGRATION flag even 2886 * if mapcount goes down to 0. We can do remap successfully 2887 * without charging it again. 2888 * 2889 * C) 2890 * The "old" page is under lock_page() until the end of 2891 * migration, so, the old page itself will not be swapped-out. 2892 * If the new page is swapped out before end_migraton, our 2893 * hook to usual swap-out path will catch the event. 2894 */ 2895 if (PageAnon(page)) 2896 SetPageCgroupMigration(pc); 2897 } 2898 unlock_page_cgroup(pc); 2899 /* 2900 * If the page is not charged at this point, 2901 * we return here. 2902 */ 2903 if (!mem) 2904 return 0; 2905 2906 *ptr = mem; 2907 ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, ptr, false, PAGE_SIZE); 2908 css_put(&mem->css);/* drop extra refcnt */ 2909 if (ret || *ptr == NULL) { 2910 if (PageAnon(page)) { 2911 lock_page_cgroup(pc); 2912 ClearPageCgroupMigration(pc); 2913 unlock_page_cgroup(pc); 2914 /* 2915 * The old page may be fully unmapped while we kept it. 2916 */ 2917 mem_cgroup_uncharge_page(page); 2918 } 2919 return -ENOMEM; 2920 } 2921 /* 2922 * We charge new page before it's used/mapped. So, even if unlock_page() 2923 * is called before end_migration, we can catch all events on this new 2924 * page. In the case new page is migrated but not remapped, new page's 2925 * mapcount will be finally 0 and we call uncharge in end_migration(). 2926 */ 2927 pc = lookup_page_cgroup(newpage); 2928 if (PageAnon(page)) 2929 ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED; 2930 else if (page_is_file_cache(page)) 2931 ctype = MEM_CGROUP_CHARGE_TYPE_CACHE; 2932 else 2933 ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM; 2934 __mem_cgroup_commit_charge(mem, pc, ctype, PAGE_SIZE); 2935 return ret; 2936} 2937 2938/* remove redundant charge if migration failed*/ 2939void mem_cgroup_end_migration(struct mem_cgroup *mem, 2940 struct page *oldpage, struct page *newpage, bool migration_ok) 2941{ 2942 struct page *used, *unused; 2943 struct page_cgroup *pc; 2944 2945 if (!mem) 2946 return; 2947 /* blocks rmdir() */ 2948 cgroup_exclude_rmdir(&mem->css); 2949 if (!migration_ok) { 2950 used = oldpage; 2951 unused = newpage; 2952 } else { 2953 used = newpage; 2954 unused = oldpage; 2955 } 2956 /* 2957 * We disallowed uncharge of pages under migration because mapcount 2958 * of the page goes down to zero, temporarly. 2959 * Clear the flag and check the page should be charged. 2960 */ 2961 pc = lookup_page_cgroup(oldpage); 2962 lock_page_cgroup(pc); 2963 ClearPageCgroupMigration(pc); 2964 unlock_page_cgroup(pc); 2965 2966 __mem_cgroup_uncharge_common(unused, MEM_CGROUP_CHARGE_TYPE_FORCE); 2967 2968 /* 2969 * If a page is a file cache, radix-tree replacement is very atomic 2970 * and we can skip this check. When it was an Anon page, its mapcount 2971 * goes down to 0. But because we added MIGRATION flage, it's not 2972 * uncharged yet. There are several case but page->mapcount check 2973 * and USED bit check in mem_cgroup_uncharge_page() will do enough 2974 * check. (see prepare_charge() also) 2975 */ 2976 if (PageAnon(used)) 2977 mem_cgroup_uncharge_page(used); 2978 /* 2979 * At migration, we may charge account against cgroup which has no 2980 * tasks. 2981 * So, rmdir()->pre_destroy() can be called while we do this charge. 2982 * In that case, we need to call pre_destroy() again. check it here. 2983 */ 2984 cgroup_release_and_wakeup_rmdir(&mem->css); 2985} 2986 2987/* 2988 * A call to try to shrink memory usage on charge failure at shmem's swapin. 2989 * Calling hierarchical_reclaim is not enough because we should update 2990 * last_oom_jiffies to prevent pagefault_out_of_memory from invoking global OOM. 2991 * Moreover considering hierarchy, we should reclaim from the mem_over_limit, 2992 * not from the memcg which this page would be charged to. 2993 * try_charge_swapin does all of these works properly. 2994 */ 2995int mem_cgroup_shmem_charge_fallback(struct page *page, 2996 struct mm_struct *mm, 2997 gfp_t gfp_mask) 2998{ 2999 struct mem_cgroup *mem = NULL; 3000 int ret; 3001 3002 if (mem_cgroup_disabled()) 3003 return 0; 3004 3005 ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem); 3006 if (!ret) 3007 mem_cgroup_cancel_charge_swapin(mem); /* it does !mem check */ 3008 3009 return ret; 3010} 3011 3012static DEFINE_MUTEX(set_limit_mutex); 3013 3014static int mem_cgroup_resize_limit(struct mem_cgroup *memcg, 3015 unsigned long long val) 3016{ 3017 int retry_count; 3018 u64 memswlimit, memlimit; 3019 int ret = 0; 3020 int children = mem_cgroup_count_children(memcg); 3021 u64 curusage, oldusage; 3022 int enlarge; 3023 3024 /* 3025 * For keeping hierarchical_reclaim simple, how long we should retry 3026 * is depends on callers. We set our retry-count to be function 3027 * of # of children which we should visit in this loop. 3028 */ 3029 retry_count = MEM_CGROUP_RECLAIM_RETRIES * children; 3030 3031 oldusage = res_counter_read_u64(&memcg->res, RES_USAGE); 3032 3033 enlarge = 0; 3034 while (retry_count) { 3035 if (signal_pending(current)) { 3036 ret = -EINTR; 3037 break; 3038 } 3039 /* 3040 * Rather than hide all in some function, I do this in 3041 * open coded manner. You see what this really does. 3042 * We have to guarantee mem->res.limit < mem->memsw.limit. 3043 */ 3044 mutex_lock(&set_limit_mutex); 3045 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 3046 if (memswlimit < val) { 3047 ret = -EINVAL; 3048 mutex_unlock(&set_limit_mutex); 3049 break; 3050 } 3051 3052 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT); 3053 if (memlimit < val) 3054 enlarge = 1; 3055 3056 ret = res_counter_set_limit(&memcg->res, val); 3057 if (!ret) { 3058 if (memswlimit == val) 3059 memcg->memsw_is_minimum = true; 3060 else 3061 memcg->memsw_is_minimum = false; 3062 } 3063 mutex_unlock(&set_limit_mutex); 3064 3065 if (!ret) 3066 break; 3067 3068 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL, 3069 MEM_CGROUP_RECLAIM_SHRINK); 3070 curusage = res_counter_read_u64(&memcg->res, RES_USAGE); 3071 /* Usage is reduced ? */ 3072 if (curusage >= oldusage) 3073 retry_count--; 3074 else 3075 oldusage = curusage; 3076 } 3077 if (!ret && enlarge) 3078 memcg_oom_recover(memcg); 3079 3080 return ret; 3081} 3082 3083static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg, 3084 unsigned long long val) 3085{ 3086 int retry_count; 3087 u64 memlimit, memswlimit, oldusage, curusage; 3088 int children = mem_cgroup_count_children(memcg); 3089 int ret = -EBUSY; 3090 int enlarge = 0; 3091 3092 /* see mem_cgroup_resize_res_limit */ 3093 retry_count = children * MEM_CGROUP_RECLAIM_RETRIES; 3094 oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE); 3095 while (retry_count) { 3096 if (signal_pending(current)) { 3097 ret = -EINTR; 3098 break; 3099 } 3100 /* 3101 * Rather than hide all in some function, I do this in 3102 * open coded manner. You see what this really does. 3103 * We have to guarantee mem->res.limit < mem->memsw.limit. 3104 */ 3105 mutex_lock(&set_limit_mutex); 3106 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT); 3107 if (memlimit > val) { 3108 ret = -EINVAL; 3109 mutex_unlock(&set_limit_mutex); 3110 break; 3111 } 3112 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 3113 if (memswlimit < val) 3114 enlarge = 1; 3115 ret = res_counter_set_limit(&memcg->memsw, val); 3116 if (!ret) { 3117 if (memlimit == val) 3118 memcg->memsw_is_minimum = true; 3119 else 3120 memcg->memsw_is_minimum = false; 3121 } 3122 mutex_unlock(&set_limit_mutex); 3123 3124 if (!ret) 3125 break; 3126 3127 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL, 3128 MEM_CGROUP_RECLAIM_NOSWAP | 3129 MEM_CGROUP_RECLAIM_SHRINK); 3130 curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE); 3131 /* Usage is reduced ? */ 3132 if (curusage >= oldusage) 3133 retry_count--; 3134 else 3135 oldusage = curusage; 3136 } 3137 if (!ret && enlarge) 3138 memcg_oom_recover(memcg); 3139 return ret; 3140} 3141 3142unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order, 3143 gfp_t gfp_mask) 3144{ 3145 unsigned long nr_reclaimed = 0; 3146 struct mem_cgroup_per_zone *mz, *next_mz = NULL; 3147 unsigned long reclaimed; 3148 int loop = 0; 3149 struct mem_cgroup_tree_per_zone *mctz; 3150 unsigned long long excess; 3151 3152 if (order > 0) 3153 return 0; 3154 3155 mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone)); 3156 /* 3157 * This loop can run a while, specially if mem_cgroup's continuously 3158 * keep exceeding their soft limit and putting the system under 3159 * pressure 3160 */ 3161 do { 3162 if (next_mz) 3163 mz = next_mz; 3164 else 3165 mz = mem_cgroup_largest_soft_limit_node(mctz); 3166 if (!mz) 3167 break; 3168 3169 reclaimed = mem_cgroup_hierarchical_reclaim(mz->mem, zone, 3170 gfp_mask, 3171 MEM_CGROUP_RECLAIM_SOFT); 3172 nr_reclaimed += reclaimed; 3173 spin_lock(&mctz->lock); 3174 3175 /* 3176 * If we failed to reclaim anything from this memory cgroup 3177 * it is time to move on to the next cgroup 3178 */ 3179 next_mz = NULL; 3180 if (!reclaimed) { 3181 do { 3182 /* 3183 * Loop until we find yet another one. 3184 * 3185 * By the time we get the soft_limit lock 3186 * again, someone might have aded the 3187 * group back on the RB tree. Iterate to 3188 * make sure we get a different mem. 3189 * mem_cgroup_largest_soft_limit_node returns 3190 * NULL if no other cgroup is present on 3191 * the tree 3192 */ 3193 next_mz = 3194 __mem_cgroup_largest_soft_limit_node(mctz); 3195 if (next_mz == mz) { 3196 css_put(&next_mz->mem->css); 3197 next_mz = NULL; 3198 } else /* next_mz == NULL or other memcg */ 3199 break; 3200 } while (1); 3201 } 3202 __mem_cgroup_remove_exceeded(mz->mem, mz, mctz); 3203 excess = res_counter_soft_limit_excess(&mz->mem->res); 3204 /* 3205 * One school of thought says that we should not add 3206 * back the node to the tree if reclaim returns 0. 3207 * But our reclaim could return 0, simply because due 3208 * to priority we are exposing a smaller subset of 3209 * memory to reclaim from. Consider this as a longer 3210 * term TODO. 3211 */ 3212 /* If excess == 0, no tree ops */ 3213 __mem_cgroup_insert_exceeded(mz->mem, mz, mctz, excess); 3214 spin_unlock(&mctz->lock); 3215 css_put(&mz->mem->css); 3216 loop++; 3217 /* 3218 * Could not reclaim anything and there are no more 3219 * mem cgroups to try or we seem to be looping without 3220 * reclaiming anything. 3221 */ 3222 if (!nr_reclaimed && 3223 (next_mz == NULL || 3224 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS)) 3225 break; 3226 } while (!nr_reclaimed); 3227 if (next_mz) 3228 css_put(&next_mz->mem->css); 3229 return nr_reclaimed; 3230} 3231 3232/* 3233 * This routine traverse page_cgroup in given list and drop them all. 3234 * *And* this routine doesn't reclaim page itself, just removes page_cgroup. 3235 */ 3236static int mem_cgroup_force_empty_list(struct mem_cgroup *mem, 3237 int node, int zid, enum lru_list lru) 3238{ 3239 struct zone *zone; 3240 struct mem_cgroup_per_zone *mz; 3241 struct page_cgroup *pc, *busy; 3242 unsigned long flags, loop; 3243 struct list_head *list; 3244 int ret = 0; 3245 3246 zone = &NODE_DATA(node)->node_zones[zid]; 3247 mz = mem_cgroup_zoneinfo(mem, node, zid); 3248 list = &mz->lists[lru]; 3249 3250 loop = MEM_CGROUP_ZSTAT(mz, lru); 3251 /* give some margin against EBUSY etc...*/ 3252 loop += 256; 3253 busy = NULL; 3254 while (loop--) { 3255 ret = 0; 3256 spin_lock_irqsave(&zone->lru_lock, flags); 3257 if (list_empty(list)) { 3258 spin_unlock_irqrestore(&zone->lru_lock, flags); 3259 break; 3260 } 3261 pc = list_entry(list->prev, struct page_cgroup, lru); 3262 if (busy == pc) { 3263 list_move(&pc->lru, list); 3264 busy = NULL; 3265 spin_unlock_irqrestore(&zone->lru_lock, flags); 3266 continue; 3267 } 3268 spin_unlock_irqrestore(&zone->lru_lock, flags); 3269 3270 ret = mem_cgroup_move_parent(pc, mem, GFP_KERNEL); 3271 if (ret == -ENOMEM) 3272 break; 3273 3274 if (ret == -EBUSY || ret == -EINVAL) { 3275 /* found lock contention or "pc" is obsolete. */ 3276 busy = pc; 3277 cond_resched(); 3278 } else 3279 busy = NULL; 3280 } 3281 3282 if (!ret && !list_empty(list)) 3283 return -EBUSY; 3284 return ret; 3285} 3286 3287/* 3288 * make mem_cgroup's charge to be 0 if there is no task. 3289 * This enables deleting this mem_cgroup. 3290 */ 3291static int mem_cgroup_force_empty(struct mem_cgroup *mem, bool free_all) 3292{ 3293 int ret; 3294 int node, zid, shrink; 3295 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES; 3296 struct cgroup *cgrp = mem->css.cgroup; 3297 3298 css_get(&mem->css); 3299 3300 shrink = 0; 3301 /* should free all ? */ 3302 if (free_all) 3303 goto try_to_free; 3304move_account: 3305 do { 3306 ret = -EBUSY; 3307 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children)) 3308 goto out; 3309 ret = -EINTR; 3310 if (signal_pending(current)) 3311 goto out; 3312 /* This is for making all *used* pages to be on LRU. */ 3313 lru_add_drain_all(); 3314 drain_all_stock_sync(); 3315 ret = 0; 3316 mem_cgroup_start_move(mem); 3317 for_each_node_state(node, N_HIGH_MEMORY) { 3318 for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) { 3319 enum lru_list l; 3320 for_each_lru(l) { 3321 ret = mem_cgroup_force_empty_list(mem, 3322 node, zid, l); 3323 if (ret) 3324 break; 3325 } 3326 } 3327 if (ret) 3328 break; 3329 } 3330 mem_cgroup_end_move(mem); 3331 memcg_oom_recover(mem); 3332 /* it seems parent cgroup doesn't have enough mem */ 3333 if (ret == -ENOMEM) 3334 goto try_to_free; 3335 cond_resched(); 3336 /* "ret" should also be checked to ensure all lists are empty. */ 3337 } while (mem->res.usage > 0 || ret); 3338out: 3339 css_put(&mem->css); 3340 return ret; 3341 3342try_to_free: 3343 /* returns EBUSY if there is a task or if we come here twice. */ 3344 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children) || shrink) { 3345 ret = -EBUSY; 3346 goto out; 3347 } 3348 /* we call try-to-free pages for make this cgroup empty */ 3349 lru_add_drain_all(); 3350 /* try to free all pages in this cgroup */ 3351 shrink = 1; 3352 while (nr_retries && mem->res.usage > 0) { 3353 int progress; 3354 3355 if (signal_pending(current)) { 3356 ret = -EINTR; 3357 goto out; 3358 } 3359 progress = try_to_free_mem_cgroup_pages(mem, GFP_KERNEL, 3360 false, get_swappiness(mem)); 3361 if (!progress) { 3362 nr_retries--; 3363 /* maybe some writeback is necessary */ 3364 congestion_wait(BLK_RW_ASYNC, HZ/10); 3365 } 3366 3367 } 3368 lru_add_drain(); 3369 /* try move_account...there may be some *locked* pages. */ 3370 goto move_account; 3371} 3372 3373int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event) 3374{ 3375 return mem_cgroup_force_empty(mem_cgroup_from_cont(cont), true); 3376} 3377 3378 3379static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft) 3380{ 3381 return mem_cgroup_from_cont(cont)->use_hierarchy; 3382} 3383 3384static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft, 3385 u64 val) 3386{ 3387 int retval = 0; 3388 struct mem_cgroup *mem = mem_cgroup_from_cont(cont); 3389 struct cgroup *parent = cont->parent; 3390 struct mem_cgroup *parent_mem = NULL; 3391 3392 if (parent) 3393 parent_mem = mem_cgroup_from_cont(parent); 3394 3395 cgroup_lock(); 3396 /* 3397 * If parent's use_hierarchy is set, we can't make any modifications 3398 * in the child subtrees. If it is unset, then the change can 3399 * occur, provided the current cgroup has no children. 3400 * 3401 * For the root cgroup, parent_mem is NULL, we allow value to be 3402 * set if there are no children. 3403 */ 3404 if ((!parent_mem || !parent_mem->use_hierarchy) && 3405 (val == 1 || val == 0)) { 3406 if (list_empty(&cont->children)) 3407 mem->use_hierarchy = val; 3408 else 3409 retval = -EBUSY; 3410 } else 3411 retval = -EINVAL; 3412 cgroup_unlock(); 3413 3414 return retval; 3415} 3416 3417 3418static u64 mem_cgroup_get_recursive_idx_stat(struct mem_cgroup *mem, 3419 enum mem_cgroup_stat_index idx) 3420{ 3421 struct mem_cgroup *iter; 3422 s64 val = 0; 3423 3424 /* each per cpu's value can be minus.Then, use s64 */ 3425 for_each_mem_cgroup_tree(iter, mem) 3426 val += mem_cgroup_read_stat(iter, idx); 3427 3428 if (val < 0) /* race ? */ 3429 val = 0; 3430 return val; 3431} 3432 3433static inline u64 mem_cgroup_usage(struct mem_cgroup *mem, bool swap) 3434{ 3435 u64 val; 3436 3437 if (!mem_cgroup_is_root(mem)) { 3438 if (!swap) 3439 return res_counter_read_u64(&mem->res, RES_USAGE); 3440 else 3441 return res_counter_read_u64(&mem->memsw, RES_USAGE); 3442 } 3443 3444 val = mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_CACHE); 3445 val += mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_RSS); 3446 3447 if (swap) 3448 val += mem_cgroup_get_recursive_idx_stat(mem, 3449 MEM_CGROUP_STAT_SWAPOUT); 3450 3451 return val << PAGE_SHIFT; 3452} 3453 3454static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft) 3455{ 3456 struct mem_cgroup *mem = mem_cgroup_from_cont(cont); 3457 u64 val; 3458 int type, name; 3459 3460 type = MEMFILE_TYPE(cft->private); 3461 name = MEMFILE_ATTR(cft->private); 3462 switch (type) { 3463 case _MEM: 3464 if (name == RES_USAGE) 3465 val = mem_cgroup_usage(mem, false); 3466 else 3467 val = res_counter_read_u64(&mem->res, name); 3468 break; 3469 case _MEMSWAP: 3470 if (name == RES_USAGE) 3471 val = mem_cgroup_usage(mem, true); 3472 else 3473 val = res_counter_read_u64(&mem->memsw, name); 3474 break; 3475 default: 3476 BUG(); 3477 break; 3478 } 3479 return val; 3480} 3481/* 3482 * The user of this function is... 3483 * RES_LIMIT. 3484 */ 3485static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft, 3486 const char *buffer) 3487{ 3488 struct mem_cgroup *memcg = mem_cgroup_from_cont(cont); 3489 int type, name; 3490 unsigned long long val; 3491 int ret; 3492 3493 type = MEMFILE_TYPE(cft->private); 3494 name = MEMFILE_ATTR(cft->private); 3495 switch (name) { 3496 case RES_LIMIT: 3497 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */ 3498 ret = -EINVAL; 3499 break; 3500 } 3501 /* This function does all necessary parse...reuse it */ 3502 ret = res_counter_memparse_write_strategy(buffer, &val); 3503 if (ret) 3504 break; 3505 if (type == _MEM) 3506 ret = mem_cgroup_resize_limit(memcg, val); 3507 else 3508 ret = mem_cgroup_resize_memsw_limit(memcg, val); 3509 break; 3510 case RES_SOFT_LIMIT: 3511 ret = res_counter_memparse_write_strategy(buffer, &val); 3512 if (ret) 3513 break; 3514 /* 3515 * For memsw, soft limits are hard to implement in terms 3516 * of semantics, for now, we support soft limits for 3517 * control without swap 3518 */ 3519 if (type == _MEM) 3520 ret = res_counter_set_soft_limit(&memcg->res, val); 3521 else 3522 ret = -EINVAL; 3523 break; 3524 default: 3525 ret = -EINVAL; /* should be BUG() ? */ 3526 break; 3527 } 3528 return ret; 3529} 3530 3531static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg, 3532 unsigned long long *mem_limit, unsigned long long *memsw_limit) 3533{ 3534 struct cgroup *cgroup; 3535 unsigned long long min_limit, min_memsw_limit, tmp; 3536 3537 min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT); 3538 min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 3539 cgroup = memcg->css.cgroup; 3540 if (!memcg->use_hierarchy) 3541 goto out; 3542 3543 while (cgroup->parent) { 3544 cgroup = cgroup->parent; 3545 memcg = mem_cgroup_from_cont(cgroup); 3546 if (!memcg->use_hierarchy) 3547 break; 3548 tmp = res_counter_read_u64(&memcg->res, RES_LIMIT); 3549 min_limit = min(min_limit, tmp); 3550 tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 3551 min_memsw_limit = min(min_memsw_limit, tmp); 3552 } 3553out: 3554 *mem_limit = min_limit; 3555 *memsw_limit = min_memsw_limit; 3556 return; 3557} 3558 3559static int mem_cgroup_reset(struct cgroup *cont, unsigned int event) 3560{ 3561 struct mem_cgroup *mem; 3562 int type, name; 3563 3564 mem = mem_cgroup_from_cont(cont); 3565 type = MEMFILE_TYPE(event); 3566 name = MEMFILE_ATTR(event); 3567 switch (name) { 3568 case RES_MAX_USAGE: 3569 if (type == _MEM) 3570 res_counter_reset_max(&mem->res); 3571 else 3572 res_counter_reset_max(&mem->memsw); 3573 break; 3574 case RES_FAILCNT: 3575 if (type == _MEM) 3576 res_counter_reset_failcnt(&mem->res); 3577 else 3578 res_counter_reset_failcnt(&mem->memsw); 3579 break; 3580 } 3581 3582 return 0; 3583} 3584 3585static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp, 3586 struct cftype *cft) 3587{ 3588 return mem_cgroup_from_cont(cgrp)->move_charge_at_immigrate; 3589} 3590 3591#ifdef CONFIG_MMU 3592static int mem_cgroup_move_charge_write(struct cgroup *cgrp, 3593 struct cftype *cft, u64 val) 3594{ 3595 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp); 3596 3597 if (val >= (1 << NR_MOVE_TYPE)) 3598 return -EINVAL; 3599 /* 3600 * We check this value several times in both in can_attach() and 3601 * attach(), so we need cgroup lock to prevent this value from being 3602 * inconsistent. 3603 */ 3604 cgroup_lock(); 3605 mem->move_charge_at_immigrate = val; 3606 cgroup_unlock(); 3607 3608 return 0; 3609} 3610#else 3611static int mem_cgroup_move_charge_write(struct cgroup *cgrp, 3612 struct cftype *cft, u64 val) 3613{ 3614 return -ENOSYS; 3615} 3616#endif 3617 3618 3619/* For read statistics */ 3620enum { 3621 MCS_CACHE, 3622 MCS_RSS, 3623 MCS_FILE_MAPPED, 3624 MCS_PGPGIN, 3625 MCS_PGPGOUT, 3626 MCS_SWAP, 3627 MCS_INACTIVE_ANON, 3628 MCS_ACTIVE_ANON, 3629 MCS_INACTIVE_FILE, 3630 MCS_ACTIVE_FILE, 3631 MCS_UNEVICTABLE, 3632 NR_MCS_STAT, 3633}; 3634 3635struct mcs_total_stat { 3636 s64 stat[NR_MCS_STAT]; 3637}; 3638 3639struct { 3640 char *local_name; 3641 char *total_name; 3642} memcg_stat_strings[NR_MCS_STAT] = { 3643 {"cache", "total_cache"}, 3644 {"rss", "total_rss"}, 3645 {"mapped_file", "total_mapped_file"}, 3646 {"pgpgin", "total_pgpgin"}, 3647 {"pgpgout", "total_pgpgout"}, 3648 {"swap", "total_swap"}, 3649 {"inactive_anon", "total_inactive_anon"}, 3650 {"active_anon", "total_active_anon"}, 3651 {"inactive_file", "total_inactive_file"}, 3652 {"active_file", "total_active_file"}, 3653 {"unevictable", "total_unevictable"} 3654}; 3655 3656 3657static void 3658mem_cgroup_get_local_stat(struct mem_cgroup *mem, struct mcs_total_stat *s) 3659{ 3660 s64 val; 3661 3662 /* per cpu stat */ 3663 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE); 3664 s->stat[MCS_CACHE] += val * PAGE_SIZE; 3665 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS); 3666 s->stat[MCS_RSS] += val * PAGE_SIZE; 3667 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_FILE_MAPPED); 3668 s->stat[MCS_FILE_MAPPED] += val * PAGE_SIZE; 3669 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGIN_COUNT); 3670 s->stat[MCS_PGPGIN] += val; 3671 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGOUT_COUNT); 3672 s->stat[MCS_PGPGOUT] += val; 3673 if (do_swap_account) { 3674 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_SWAPOUT); 3675 s->stat[MCS_SWAP] += val * PAGE_SIZE; 3676 } 3677 3678 /* per zone stat */ 3679 val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_ANON); 3680 s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE; 3681 val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_ANON); 3682 s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE; 3683 val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_FILE); 3684 s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE; 3685 val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_FILE); 3686 s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE; 3687 val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE); 3688 s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE; 3689} 3690 3691static void 3692mem_cgroup_get_total_stat(struct mem_cgroup *mem, struct mcs_total_stat *s) 3693{ 3694 struct mem_cgroup *iter; 3695 3696 for_each_mem_cgroup_tree(iter, mem) 3697 mem_cgroup_get_local_stat(iter, s); 3698} 3699 3700static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft, 3701 struct cgroup_map_cb *cb) 3702{ 3703 struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont); 3704 struct mcs_total_stat mystat; 3705 int i; 3706 3707 memset(&mystat, 0, sizeof(mystat)); 3708 mem_cgroup_get_local_stat(mem_cont, &mystat); 3709 3710 for (i = 0; i < NR_MCS_STAT; i++) { 3711 if (i == MCS_SWAP && !do_swap_account) 3712 continue; 3713 cb->fill(cb, memcg_stat_strings[i].local_name, mystat.stat[i]); 3714 } 3715 3716 /* Hierarchical information */ 3717 { 3718 unsigned long long limit, memsw_limit; 3719 memcg_get_hierarchical_limit(mem_cont, &limit, &memsw_limit); 3720 cb->fill(cb, "hierarchical_memory_limit", limit); 3721 if (do_swap_account) 3722 cb->fill(cb, "hierarchical_memsw_limit", memsw_limit); 3723 } 3724 3725 memset(&mystat, 0, sizeof(mystat)); 3726 mem_cgroup_get_total_stat(mem_cont, &mystat); 3727 for (i = 0; i < NR_MCS_STAT; i++) { 3728 if (i == MCS_SWAP && !do_swap_account) 3729 continue; 3730 cb->fill(cb, memcg_stat_strings[i].total_name, mystat.stat[i]); 3731 } 3732 3733#ifdef CONFIG_DEBUG_VM 3734 cb->fill(cb, "inactive_ratio", calc_inactive_ratio(mem_cont, NULL)); 3735 3736 { 3737 int nid, zid; 3738 struct mem_cgroup_per_zone *mz; 3739 unsigned long recent_rotated[2] = {0, 0}; 3740 unsigned long recent_scanned[2] = {0, 0}; 3741 3742 for_each_online_node(nid) 3743 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 3744 mz = mem_cgroup_zoneinfo(mem_cont, nid, zid); 3745 3746 recent_rotated[0] += 3747 mz->reclaim_stat.recent_rotated[0]; 3748 recent_rotated[1] += 3749 mz->reclaim_stat.recent_rotated[1]; 3750 recent_scanned[0] += 3751 mz->reclaim_stat.recent_scanned[0]; 3752 recent_scanned[1] += 3753 mz->reclaim_stat.recent_scanned[1]; 3754 } 3755 cb->fill(cb, "recent_rotated_anon", recent_rotated[0]); 3756 cb->fill(cb, "recent_rotated_file", recent_rotated[1]); 3757 cb->fill(cb, "recent_scanned_anon", recent_scanned[0]); 3758 cb->fill(cb, "recent_scanned_file", recent_scanned[1]); 3759 } 3760#endif 3761 3762 return 0; 3763} 3764 3765static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft) 3766{ 3767 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); 3768 3769 return get_swappiness(memcg); 3770} 3771 3772static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft, 3773 u64 val) 3774{ 3775 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); 3776 struct mem_cgroup *parent; 3777 3778 if (val > 100) 3779 return -EINVAL; 3780 3781 if (cgrp->parent == NULL) 3782 return -EINVAL; 3783 3784 parent = mem_cgroup_from_cont(cgrp->parent); 3785 3786 cgroup_lock(); 3787 3788 /* If under hierarchy, only empty-root can set this value */ 3789 if ((parent->use_hierarchy) || 3790 (memcg->use_hierarchy && !list_empty(&cgrp->children))) { 3791 cgroup_unlock(); 3792 return -EINVAL; 3793 } 3794 3795 spin_lock(&memcg->reclaim_param_lock); 3796 memcg->swappiness = val; 3797 spin_unlock(&memcg->reclaim_param_lock); 3798 3799 cgroup_unlock(); 3800 3801 return 0; 3802} 3803 3804static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap) 3805{ 3806 struct mem_cgroup_threshold_ary *t; 3807 u64 usage; 3808 int i; 3809 3810 rcu_read_lock(); 3811 if (!swap) 3812 t = rcu_dereference(memcg->thresholds.primary); 3813 else 3814 t = rcu_dereference(memcg->memsw_thresholds.primary); 3815 3816 if (!t) 3817 goto unlock; 3818 3819 usage = mem_cgroup_usage(memcg, swap); 3820 3821 /* 3822 * current_threshold points to threshold just below usage. 3823 * If it's not true, a threshold was crossed after last 3824 * call of __mem_cgroup_threshold(). 3825 */ 3826 i = t->current_threshold; 3827 3828 /* 3829 * Iterate backward over array of thresholds starting from 3830 * current_threshold and check if a threshold is crossed. 3831 * If none of thresholds below usage is crossed, we read 3832 * only one element of the array here. 3833 */ 3834 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--) 3835 eventfd_signal(t->entries[i].eventfd, 1); 3836 3837 /* i = current_threshold + 1 */ 3838 i++; 3839 3840 /* 3841 * Iterate forward over array of thresholds starting from 3842 * current_threshold+1 and check if a threshold is crossed. 3843 * If none of thresholds above usage is crossed, we read 3844 * only one element of the array here. 3845 */ 3846 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++) 3847 eventfd_signal(t->entries[i].eventfd, 1); 3848 3849 /* Update current_threshold */ 3850 t->current_threshold = i - 1; 3851unlock: 3852 rcu_read_unlock(); 3853} 3854 3855static void mem_cgroup_threshold(struct mem_cgroup *memcg) 3856{ 3857 while (memcg) { 3858 __mem_cgroup_threshold(memcg, false); 3859 if (do_swap_account) 3860 __mem_cgroup_threshold(memcg, true); 3861 3862 memcg = parent_mem_cgroup(memcg); 3863 } 3864} 3865 3866static int compare_thresholds(const void *a, const void *b) 3867{ 3868 const struct mem_cgroup_threshold *_a = a; 3869 const struct mem_cgroup_threshold *_b = b; 3870 3871 return _a->threshold - _b->threshold; 3872} 3873 3874static int mem_cgroup_oom_notify_cb(struct mem_cgroup *mem) 3875{ 3876 struct mem_cgroup_eventfd_list *ev; 3877 3878 list_for_each_entry(ev, &mem->oom_notify, list) 3879 eventfd_signal(ev->eventfd, 1); 3880 return 0; 3881} 3882 3883static void mem_cgroup_oom_notify(struct mem_cgroup *mem) 3884{ 3885 struct mem_cgroup *iter; 3886 3887 for_each_mem_cgroup_tree(iter, mem) 3888 mem_cgroup_oom_notify_cb(iter); 3889} 3890 3891static int mem_cgroup_usage_register_event(struct cgroup *cgrp, 3892 struct cftype *cft, struct eventfd_ctx *eventfd, const char *args) 3893{ 3894 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); 3895 struct mem_cgroup_thresholds *thresholds; 3896 struct mem_cgroup_threshold_ary *new; 3897 int type = MEMFILE_TYPE(cft->private); 3898 u64 threshold, usage; 3899 int i, size, ret; 3900 3901 ret = res_counter_memparse_write_strategy(args, &threshold); 3902 if (ret) 3903 return ret; 3904 3905 mutex_lock(&memcg->thresholds_lock); 3906 3907 if (type == _MEM) 3908 thresholds = &memcg->thresholds; 3909 else if (type == _MEMSWAP) 3910 thresholds = &memcg->memsw_thresholds; 3911 else 3912 BUG(); 3913 3914 usage = mem_cgroup_usage(memcg, type == _MEMSWAP); 3915 3916 /* Check if a threshold crossed before adding a new one */ 3917 if (thresholds->primary) 3918 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 3919 3920 size = thresholds->primary ? thresholds->primary->size + 1 : 1; 3921 3922 /* Allocate memory for new array of thresholds */ 3923 new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold), 3924 GFP_KERNEL); 3925 if (!new) { 3926 ret = -ENOMEM; 3927 goto unlock; 3928 } 3929 new->size = size; 3930 3931 /* Copy thresholds (if any) to new array */ 3932 if (thresholds->primary) { 3933 memcpy(new->entries, thresholds->primary->entries, (size - 1) * 3934 sizeof(struct mem_cgroup_threshold)); 3935 } 3936 3937 /* Add new threshold */ 3938 new->entries[size - 1].eventfd = eventfd; 3939 new->entries[size - 1].threshold = threshold; 3940 3941 /* Sort thresholds. Registering of new threshold isn't time-critical */ 3942 sort(new->entries, size, sizeof(struct mem_cgroup_threshold), 3943 compare_thresholds, NULL); 3944 3945 /* Find current threshold */ 3946 new->current_threshold = -1; 3947 for (i = 0; i < size; i++) { 3948 if (new->entries[i].threshold < usage) { 3949 /* 3950 * new->current_threshold will not be used until 3951 * rcu_assign_pointer(), so it's safe to increment 3952 * it here. 3953 */ 3954 ++new->current_threshold; 3955 } 3956 } 3957 3958 /* Free old spare buffer and save old primary buffer as spare */ 3959 kfree(thresholds->spare); 3960 thresholds->spare = thresholds->primary; 3961 3962 rcu_assign_pointer(thresholds->primary, new); 3963 3964 /* To be sure that nobody uses thresholds */ 3965 synchronize_rcu(); 3966 3967unlock: 3968 mutex_unlock(&memcg->thresholds_lock); 3969 3970 return ret; 3971} 3972 3973static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp, 3974 struct cftype *cft, struct eventfd_ctx *eventfd) 3975{ 3976 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); 3977 struct mem_cgroup_thresholds *thresholds; 3978 struct mem_cgroup_threshold_ary *new; 3979 int type = MEMFILE_TYPE(cft->private); 3980 u64 usage; 3981 int i, j, size; 3982 3983 mutex_lock(&memcg->thresholds_lock); 3984 if (type == _MEM) 3985 thresholds = &memcg->thresholds; 3986 else if (type == _MEMSWAP) 3987 thresholds = &memcg->memsw_thresholds; 3988 else 3989 BUG(); 3990 3991 /* 3992 * Something went wrong if we trying to unregister a threshold 3993 * if we don't have thresholds 3994 */ 3995 BUG_ON(!thresholds); 3996 3997 usage = mem_cgroup_usage(memcg, type == _MEMSWAP); 3998 3999 /* Check if a threshold crossed before removing */ 4000 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 4001 4002 /* Calculate new number of threshold */ 4003 size = 0; 4004 for (i = 0; i < thresholds->primary->size; i++) { 4005 if (thresholds->primary->entries[i].eventfd != eventfd) 4006 size++; 4007 } 4008 4009 new = thresholds->spare; 4010 4011 /* Set thresholds array to NULL if we don't have thresholds */ 4012 if (!size) { 4013 kfree(new); 4014 new = NULL; 4015 goto swap_buffers; 4016 } 4017 4018 new->size = size; 4019 4020 /* Copy thresholds and find current threshold */ 4021 new->current_threshold = -1; 4022 for (i = 0, j = 0; i < thresholds->primary->size; i++) { 4023 if (thresholds->primary->entries[i].eventfd == eventfd) 4024 continue; 4025 4026 new->entries[j] = thresholds->primary->entries[i]; 4027 if (new->entries[j].threshold < usage) { 4028 /* 4029 * new->current_threshold will not be used 4030 * until rcu_assign_pointer(), so it's safe to increment 4031 * it here. 4032 */ 4033 ++new->current_threshold; 4034 } 4035 j++; 4036 } 4037 4038swap_buffers: 4039 /* Swap primary and spare array */ 4040 thresholds->spare = thresholds->primary; 4041 rcu_assign_pointer(thresholds->primary, new); 4042 4043 /* To be sure that nobody uses thresholds */ 4044 synchronize_rcu(); 4045 4046 mutex_unlock(&memcg->thresholds_lock); 4047} 4048 4049static int mem_cgroup_oom_register_event(struct cgroup *cgrp, 4050 struct cftype *cft, struct eventfd_ctx *eventfd, const char *args) 4051{ 4052 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); 4053 struct mem_cgroup_eventfd_list *event; 4054 int type = MEMFILE_TYPE(cft->private); 4055 4056 BUG_ON(type != _OOM_TYPE); 4057 event = kmalloc(sizeof(*event), GFP_KERNEL); 4058 if (!event) 4059 return -ENOMEM; 4060 4061 mutex_lock(&memcg_oom_mutex); 4062 4063 event->eventfd = eventfd; 4064 list_add(&event->list, &memcg->oom_notify); 4065 4066 /* already in OOM ? */ 4067 if (atomic_read(&memcg->oom_lock)) 4068 eventfd_signal(eventfd, 1); 4069 mutex_unlock(&memcg_oom_mutex); 4070 4071 return 0; 4072} 4073 4074static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp, 4075 struct cftype *cft, struct eventfd_ctx *eventfd) 4076{ 4077 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp); 4078 struct mem_cgroup_eventfd_list *ev, *tmp; 4079 int type = MEMFILE_TYPE(cft->private); 4080 4081 BUG_ON(type != _OOM_TYPE); 4082 4083 mutex_lock(&memcg_oom_mutex); 4084 4085 list_for_each_entry_safe(ev, tmp, &mem->oom_notify, list) { 4086 if (ev->eventfd == eventfd) { 4087 list_del(&ev->list); 4088 kfree(ev); 4089 } 4090 } 4091 4092 mutex_unlock(&memcg_oom_mutex); 4093} 4094 4095static int mem_cgroup_oom_control_read(struct cgroup *cgrp, 4096 struct cftype *cft, struct cgroup_map_cb *cb) 4097{ 4098 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp); 4099 4100 cb->fill(cb, "oom_kill_disable", mem->oom_kill_disable); 4101 4102 if (atomic_read(&mem->oom_lock)) 4103 cb->fill(cb, "under_oom", 1); 4104 else 4105 cb->fill(cb, "under_oom", 0); 4106 return 0; 4107} 4108 4109static int mem_cgroup_oom_control_write(struct cgroup *cgrp, 4110 struct cftype *cft, u64 val) 4111{ 4112 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp); 4113 struct mem_cgroup *parent; 4114 4115 /* cannot set to root cgroup and only 0 and 1 are allowed */ 4116 if (!cgrp->parent || !((val == 0) || (val == 1))) 4117 return -EINVAL; 4118 4119 parent = mem_cgroup_from_cont(cgrp->parent); 4120 4121 cgroup_lock(); 4122 /* oom-kill-disable is a flag for subhierarchy. */ 4123 if ((parent->use_hierarchy) || 4124 (mem->use_hierarchy && !list_empty(&cgrp->children))) { 4125 cgroup_unlock(); 4126 return -EINVAL; 4127 } 4128 mem->oom_kill_disable = val; 4129 if (!val) 4130 memcg_oom_recover(mem); 4131 cgroup_unlock(); 4132 return 0; 4133} 4134 4135static struct cftype mem_cgroup_files[] = { 4136 { 4137 .name = "usage_in_bytes", 4138 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE), 4139 .read_u64 = mem_cgroup_read, 4140 .register_event = mem_cgroup_usage_register_event, 4141 .unregister_event = mem_cgroup_usage_unregister_event, 4142 }, 4143 { 4144 .name = "max_usage_in_bytes", 4145 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE), 4146 .trigger = mem_cgroup_reset, 4147 .read_u64 = mem_cgroup_read, 4148 }, 4149 { 4150 .name = "limit_in_bytes", 4151 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT), 4152 .write_string = mem_cgroup_write, 4153 .read_u64 = mem_cgroup_read, 4154 }, 4155 { 4156 .name = "soft_limit_in_bytes", 4157 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT), 4158 .write_string = mem_cgroup_write, 4159 .read_u64 = mem_cgroup_read, 4160 }, 4161 { 4162 .name = "failcnt", 4163 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT), 4164 .trigger = mem_cgroup_reset, 4165 .read_u64 = mem_cgroup_read, 4166 }, 4167 { 4168 .name = "stat", 4169 .read_map = mem_control_stat_show, 4170 }, 4171 { 4172 .name = "force_empty", 4173 .trigger = mem_cgroup_force_empty_write, 4174 }, 4175 { 4176 .name = "use_hierarchy", 4177 .write_u64 = mem_cgroup_hierarchy_write, 4178 .read_u64 = mem_cgroup_hierarchy_read, 4179 }, 4180 { 4181 .name = "swappiness", 4182 .read_u64 = mem_cgroup_swappiness_read, 4183 .write_u64 = mem_cgroup_swappiness_write, 4184 }, 4185 { 4186 .name = "move_charge_at_immigrate", 4187 .read_u64 = mem_cgroup_move_charge_read, 4188 .write_u64 = mem_cgroup_move_charge_write, 4189 }, 4190 { 4191 .name = "oom_control", 4192 .read_map = mem_cgroup_oom_control_read, 4193 .write_u64 = mem_cgroup_oom_control_write, 4194 .register_event = mem_cgroup_oom_register_event, 4195 .unregister_event = mem_cgroup_oom_unregister_event, 4196 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL), 4197 }, 4198}; 4199 4200#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP 4201static struct cftype memsw_cgroup_files[] = { 4202 { 4203 .name = "memsw.usage_in_bytes", 4204 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE), 4205 .read_u64 = mem_cgroup_read, 4206 .register_event = mem_cgroup_usage_register_event, 4207 .unregister_event = mem_cgroup_usage_unregister_event, 4208 }, 4209 { 4210 .name = "memsw.max_usage_in_bytes", 4211 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE), 4212 .trigger = mem_cgroup_reset, 4213 .read_u64 = mem_cgroup_read, 4214 }, 4215 { 4216 .name = "memsw.limit_in_bytes", 4217 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT), 4218 .write_string = mem_cgroup_write, 4219 .read_u64 = mem_cgroup_read, 4220 }, 4221 { 4222 .name = "memsw.failcnt", 4223 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT), 4224 .trigger = mem_cgroup_reset, 4225 .read_u64 = mem_cgroup_read, 4226 }, 4227}; 4228 4229static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss) 4230{ 4231 if (!do_swap_account) 4232 return 0; 4233 return cgroup_add_files(cont, ss, memsw_cgroup_files, 4234 ARRAY_SIZE(memsw_cgroup_files)); 4235}; 4236#else 4237static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss) 4238{ 4239 return 0; 4240} 4241#endif 4242 4243static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node) 4244{ 4245 struct mem_cgroup_per_node *pn; 4246 struct mem_cgroup_per_zone *mz; 4247 enum lru_list l; 4248 int zone, tmp = node; 4249 /* 4250 * This routine is called against possible nodes. 4251 * But it's BUG to call kmalloc() against offline node. 4252 * 4253 * TODO: this routine can waste much memory for nodes which will 4254 * never be onlined. It's better to use memory hotplug callback 4255 * function. 4256 */ 4257 if (!node_state(node, N_NORMAL_MEMORY)) 4258 tmp = -1; 4259 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp); 4260 if (!pn) 4261 return 1; 4262 4263 mem->info.nodeinfo[node] = pn; 4264 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 4265 mz = &pn->zoneinfo[zone]; 4266 for_each_lru(l) 4267 INIT_LIST_HEAD(&mz->lists[l]); 4268 mz->usage_in_excess = 0; 4269 mz->on_tree = false; 4270 mz->mem = mem; 4271 } 4272 return 0; 4273} 4274 4275static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node) 4276{ 4277 kfree(mem->info.nodeinfo[node]); 4278} 4279 4280static struct mem_cgroup *mem_cgroup_alloc(void) 4281{ 4282 struct mem_cgroup *mem; 4283 int size = sizeof(struct mem_cgroup); 4284 4285 /* Can be very big if MAX_NUMNODES is very big */ 4286 if (size < PAGE_SIZE) 4287 mem = kzalloc(size, GFP_KERNEL); 4288 else 4289 mem = vzalloc(size); 4290 4291 if (!mem) 4292 return NULL; 4293 4294 mem->stat = alloc_percpu(struct mem_cgroup_stat_cpu); 4295 if (!mem->stat) 4296 goto out_free; 4297 spin_lock_init(&mem->pcp_counter_lock); 4298 return mem; 4299 4300out_free: 4301 if (size < PAGE_SIZE) 4302 kfree(mem); 4303 else 4304 vfree(mem); 4305 return NULL; 4306} 4307 4308/* 4309 * At destroying mem_cgroup, references from swap_cgroup can remain. 4310 * (scanning all at force_empty is too costly...) 4311 * 4312 * Instead of clearing all references at force_empty, we remember 4313 * the number of reference from swap_cgroup and free mem_cgroup when 4314 * it goes down to 0. 4315 * 4316 * Removal of cgroup itself succeeds regardless of refs from swap. 4317 */ 4318 4319static void __mem_cgroup_free(struct mem_cgroup *mem) 4320{ 4321 int node; 4322 4323 mem_cgroup_remove_from_trees(mem); 4324 free_css_id(&mem_cgroup_subsys, &mem->css); 4325 4326 for_each_node_state(node, N_POSSIBLE) 4327 free_mem_cgroup_per_zone_info(mem, node); 4328 4329 free_percpu(mem->stat); 4330 if (sizeof(struct mem_cgroup) < PAGE_SIZE) 4331 kfree(mem); 4332 else 4333 vfree(mem); 4334} 4335 4336static void mem_cgroup_get(struct mem_cgroup *mem) 4337{ 4338 atomic_inc(&mem->refcnt); 4339} 4340 4341static void __mem_cgroup_put(struct mem_cgroup *mem, int count) 4342{ 4343 if (atomic_sub_and_test(count, &mem->refcnt)) { 4344 struct mem_cgroup *parent = parent_mem_cgroup(mem); 4345 __mem_cgroup_free(mem); 4346 if (parent) 4347 mem_cgroup_put(parent); 4348 } 4349} 4350 4351static void mem_cgroup_put(struct mem_cgroup *mem) 4352{ 4353 __mem_cgroup_put(mem, 1); 4354} 4355 4356/* 4357 * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled. 4358 */ 4359static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem) 4360{ 4361 if (!mem->res.parent) 4362 return NULL; 4363 return mem_cgroup_from_res_counter(mem->res.parent, res); 4364} 4365 4366#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP 4367static void __init enable_swap_cgroup(void) 4368{ 4369 if (!mem_cgroup_disabled() && really_do_swap_account) 4370 do_swap_account = 1; 4371} 4372#else 4373static void __init enable_swap_cgroup(void) 4374{ 4375} 4376#endif 4377 4378static int mem_cgroup_soft_limit_tree_init(void) 4379{ 4380 struct mem_cgroup_tree_per_node *rtpn; 4381 struct mem_cgroup_tree_per_zone *rtpz; 4382 int tmp, node, zone; 4383 4384 for_each_node_state(node, N_POSSIBLE) { 4385 tmp = node; 4386 if (!node_state(node, N_NORMAL_MEMORY)) 4387 tmp = -1; 4388 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp); 4389 if (!rtpn) 4390 return 1; 4391 4392 soft_limit_tree.rb_tree_per_node[node] = rtpn; 4393 4394 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 4395 rtpz = &rtpn->rb_tree_per_zone[zone]; 4396 rtpz->rb_root = RB_ROOT; 4397 spin_lock_init(&rtpz->lock); 4398 } 4399 } 4400 return 0; 4401} 4402 4403static struct cgroup_subsys_state * __ref 4404mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont) 4405{ 4406 struct mem_cgroup *mem, *parent; 4407 long error = -ENOMEM; 4408 int node; 4409 4410 mem = mem_cgroup_alloc(); 4411 if (!mem) 4412 return ERR_PTR(error); 4413 4414 for_each_node_state(node, N_POSSIBLE) 4415 if (alloc_mem_cgroup_per_zone_info(mem, node)) 4416 goto free_out; 4417 4418 /* root ? */ 4419 if (cont->parent == NULL) { 4420 int cpu; 4421 enable_swap_cgroup(); 4422 parent = NULL; 4423 root_mem_cgroup = mem; 4424 if (mem_cgroup_soft_limit_tree_init()) 4425 goto free_out; 4426 for_each_possible_cpu(cpu) { 4427 struct memcg_stock_pcp *stock = 4428 &per_cpu(memcg_stock, cpu); 4429 INIT_WORK(&stock->work, drain_local_stock); 4430 } 4431 hotcpu_notifier(memcg_cpu_hotplug_callback, 0); 4432 } else { 4433 parent = mem_cgroup_from_cont(cont->parent); 4434 mem->use_hierarchy = parent->use_hierarchy; 4435 mem->oom_kill_disable = parent->oom_kill_disable; 4436 } 4437 4438 if (parent && parent->use_hierarchy) { 4439 res_counter_init(&mem->res, &parent->res); 4440 res_counter_init(&mem->memsw, &parent->memsw); 4441 /* 4442 * We increment refcnt of the parent to ensure that we can 4443 * safely access it on res_counter_charge/uncharge. 4444 * This refcnt will be decremented when freeing this 4445 * mem_cgroup(see mem_cgroup_put). 4446 */ 4447 mem_cgroup_get(parent); 4448 } else { 4449 res_counter_init(&mem->res, NULL); 4450 res_counter_init(&mem->memsw, NULL); 4451 } 4452 mem->last_scanned_child = 0; 4453 spin_lock_init(&mem->reclaim_param_lock); 4454 INIT_LIST_HEAD(&mem->oom_notify); 4455 4456 if (parent) 4457 mem->swappiness = get_swappiness(parent); 4458 atomic_set(&mem->refcnt, 1); 4459 mem->move_charge_at_immigrate = 0; 4460 mutex_init(&mem->thresholds_lock); 4461 return &mem->css; 4462free_out: 4463 __mem_cgroup_free(mem); 4464 root_mem_cgroup = NULL; 4465 return ERR_PTR(error); 4466} 4467 4468static int mem_cgroup_pre_destroy(struct cgroup_subsys *ss, 4469 struct cgroup *cont) 4470{ 4471 struct mem_cgroup *mem = mem_cgroup_from_cont(cont); 4472 4473 return mem_cgroup_force_empty(mem, false); 4474} 4475 4476static void mem_cgroup_destroy(struct cgroup_subsys *ss, 4477 struct cgroup *cont) 4478{ 4479 struct mem_cgroup *mem = mem_cgroup_from_cont(cont); 4480 4481 mem_cgroup_put(mem); 4482} 4483 4484static int mem_cgroup_populate(struct cgroup_subsys *ss, 4485 struct cgroup *cont) 4486{ 4487 int ret; 4488 4489 ret = cgroup_add_files(cont, ss, mem_cgroup_files, 4490 ARRAY_SIZE(mem_cgroup_files)); 4491 4492 if (!ret) 4493 ret = register_memsw_files(cont, ss); 4494 return ret; 4495} 4496 4497#ifdef CONFIG_MMU 4498/* Handlers for move charge at task migration. */ 4499#define PRECHARGE_COUNT_AT_ONCE 256 4500static int mem_cgroup_do_precharge(unsigned long count) 4501{ 4502 int ret = 0; 4503 int batch_count = PRECHARGE_COUNT_AT_ONCE; 4504 struct mem_cgroup *mem = mc.to; 4505 4506 if (mem_cgroup_is_root(mem)) { 4507 mc.precharge += count; 4508 /* we don't need css_get for root */ 4509 return ret; 4510 } 4511 /* try to charge at once */ 4512 if (count > 1) { 4513 struct res_counter *dummy; 4514 /* 4515 * "mem" cannot be under rmdir() because we've already checked 4516 * by cgroup_lock_live_cgroup() that it is not removed and we 4517 * are still under the same cgroup_mutex. So we can postpone 4518 * css_get(). 4519 */ 4520 if (res_counter_charge(&mem->res, PAGE_SIZE * count, &dummy)) 4521 goto one_by_one; 4522 if (do_swap_account && res_counter_charge(&mem->memsw, 4523 PAGE_SIZE * count, &dummy)) { 4524 res_counter_uncharge(&mem->res, PAGE_SIZE * count); 4525 goto one_by_one; 4526 } 4527 mc.precharge += count; 4528 return ret; 4529 } 4530one_by_one: 4531 /* fall back to one by one charge */ 4532 while (count--) { 4533 if (signal_pending(current)) { 4534 ret = -EINTR; 4535 break; 4536 } 4537 if (!batch_count--) { 4538 batch_count = PRECHARGE_COUNT_AT_ONCE; 4539 cond_resched(); 4540 } 4541 ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem, false, 4542 PAGE_SIZE); 4543 if (ret || !mem) 4544 /* mem_cgroup_clear_mc() will do uncharge later */ 4545 return -ENOMEM; 4546 mc.precharge++; 4547 } 4548 return ret; 4549} 4550 4551/** 4552 * is_target_pte_for_mc - check a pte whether it is valid for move charge 4553 * @vma: the vma the pte to be checked belongs 4554 * @addr: the address corresponding to the pte to be checked 4555 * @ptent: the pte to be checked 4556 * @target: the pointer the target page or swap ent will be stored(can be NULL) 4557 * 4558 * Returns 4559 * 0(MC_TARGET_NONE): if the pte is not a target for move charge. 4560 * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for 4561 * move charge. if @target is not NULL, the page is stored in target->page 4562 * with extra refcnt got(Callers should handle it). 4563 * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a 4564 * target for charge migration. if @target is not NULL, the entry is stored 4565 * in target->ent. 4566 * 4567 * Called with pte lock held. 4568 */ 4569union mc_target { 4570 struct page *page; 4571 swp_entry_t ent; 4572}; 4573 4574enum mc_target_type { 4575 MC_TARGET_NONE, /* not used */ 4576 MC_TARGET_PAGE, 4577 MC_TARGET_SWAP, 4578}; 4579 4580static struct page *mc_handle_present_pte(struct vm_area_struct *vma, 4581 unsigned long addr, pte_t ptent) 4582{ 4583 struct page *page = vm_normal_page(vma, addr, ptent); 4584 4585 if (!page || !page_mapped(page)) 4586 return NULL; 4587 if (PageAnon(page)) { 4588 /* we don't move shared anon */ 4589 if (!move_anon() || page_mapcount(page) > 2) 4590 return NULL; 4591 } else if (!move_file()) 4592 /* we ignore mapcount for file pages */ 4593 return NULL; 4594 if (!get_page_unless_zero(page)) 4595 return NULL; 4596 4597 return page; 4598} 4599 4600static struct page *mc_handle_swap_pte(struct vm_area_struct *vma, 4601 unsigned long addr, pte_t ptent, swp_entry_t *entry) 4602{ 4603 int usage_count; 4604 struct page *page = NULL; 4605 swp_entry_t ent = pte_to_swp_entry(ptent); 4606 4607 if (!move_anon() || non_swap_entry(ent)) 4608 return NULL; 4609 usage_count = mem_cgroup_count_swap_user(ent, &page); 4610 if (usage_count > 1) { /* we don't move shared anon */ 4611 if (page) 4612 put_page(page); 4613 return NULL; 4614 } 4615 if (do_swap_account) 4616 entry->val = ent.val; 4617 4618 return page; 4619} 4620 4621static struct page *mc_handle_file_pte(struct vm_area_struct *vma, 4622 unsigned long addr, pte_t ptent, swp_entry_t *entry) 4623{ 4624 struct page *page = NULL; 4625 struct inode *inode; 4626 struct address_space *mapping; 4627 pgoff_t pgoff; 4628 4629 if (!vma->vm_file) /* anonymous vma */ 4630 return NULL; 4631 if (!move_file()) 4632 return NULL; 4633 4634 inode = vma->vm_file->f_path.dentry->d_inode; 4635 mapping = vma->vm_file->f_mapping; 4636 if (pte_none(ptent)) 4637 pgoff = linear_page_index(vma, addr); 4638 else /* pte_file(ptent) is true */ 4639 pgoff = pte_to_pgoff(ptent); 4640 4641 /* page is moved even if it's not RSS of this task(page-faulted). */ 4642 if (!mapping_cap_swap_backed(mapping)) { /* normal file */ 4643 page = find_get_page(mapping, pgoff); 4644 } else { /* shmem/tmpfs file. we should take account of swap too. */ 4645 swp_entry_t ent; 4646 mem_cgroup_get_shmem_target(inode, pgoff, &page, &ent); 4647 if (do_swap_account) 4648 entry->val = ent.val; 4649 } 4650 4651 return page; 4652} 4653 4654static int is_target_pte_for_mc(struct vm_area_struct *vma, 4655 unsigned long addr, pte_t ptent, union mc_target *target) 4656{ 4657 struct page *page = NULL; 4658 struct page_cgroup *pc; 4659 int ret = 0; 4660 swp_entry_t ent = { .val = 0 }; 4661 4662 if (pte_present(ptent)) 4663 page = mc_handle_present_pte(vma, addr, ptent); 4664 else if (is_swap_pte(ptent)) 4665 page = mc_handle_swap_pte(vma, addr, ptent, &ent); 4666 else if (pte_none(ptent) || pte_file(ptent)) 4667 page = mc_handle_file_pte(vma, addr, ptent, &ent); 4668 4669 if (!page && !ent.val) 4670 return 0; 4671 if (page) { 4672 pc = lookup_page_cgroup(page); 4673 /* 4674 * Do only loose check w/o page_cgroup lock. 4675 * mem_cgroup_move_account() checks the pc is valid or not under 4676 * the lock. 4677 */ 4678 if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) { 4679 ret = MC_TARGET_PAGE; 4680 if (target) 4681 target->page = page; 4682 } 4683 if (!ret || !target) 4684 put_page(page); 4685 } 4686 /* There is a swap entry and a page doesn't exist or isn't charged */ 4687 if (ent.val && !ret && 4688 css_id(&mc.from->css) == lookup_swap_cgroup(ent)) { 4689 ret = MC_TARGET_SWAP; 4690 if (target) 4691 target->ent = ent; 4692 } 4693 return ret; 4694} 4695 4696static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd, 4697 unsigned long addr, unsigned long end, 4698 struct mm_walk *walk) 4699{ 4700 struct vm_area_struct *vma = walk->private; 4701 pte_t *pte; 4702 spinlock_t *ptl; 4703 4704 VM_BUG_ON(pmd_trans_huge(*pmd)); 4705 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 4706 for (; addr != end; pte++, addr += PAGE_SIZE) 4707 if (is_target_pte_for_mc(vma, addr, *pte, NULL)) 4708 mc.precharge++; /* increment precharge temporarily */ 4709 pte_unmap_unlock(pte - 1, ptl); 4710 cond_resched(); 4711 4712 return 0; 4713} 4714 4715static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm) 4716{ 4717 unsigned long precharge; 4718 struct vm_area_struct *vma; 4719 4720 down_read(&mm->mmap_sem); 4721 for (vma = mm->mmap; vma; vma = vma->vm_next) { 4722 struct mm_walk mem_cgroup_count_precharge_walk = { 4723 .pmd_entry = mem_cgroup_count_precharge_pte_range, 4724 .mm = mm, 4725 .private = vma, 4726 }; 4727 if (is_vm_hugetlb_page(vma)) 4728 continue; 4729 walk_page_range(vma->vm_start, vma->vm_end, 4730 &mem_cgroup_count_precharge_walk); 4731 } 4732 up_read(&mm->mmap_sem); 4733 4734 precharge = mc.precharge; 4735 mc.precharge = 0; 4736 4737 return precharge; 4738} 4739 4740static int mem_cgroup_precharge_mc(struct mm_struct *mm) 4741{ 4742 unsigned long precharge = mem_cgroup_count_precharge(mm); 4743 4744 VM_BUG_ON(mc.moving_task); 4745 mc.moving_task = current; 4746 return mem_cgroup_do_precharge(precharge); 4747} 4748 4749/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */ 4750static void __mem_cgroup_clear_mc(void) 4751{ 4752 struct mem_cgroup *from = mc.from; 4753 struct mem_cgroup *to = mc.to; 4754 4755 /* we must uncharge all the leftover precharges from mc.to */ 4756 if (mc.precharge) { 4757 __mem_cgroup_cancel_charge(mc.to, mc.precharge); 4758 mc.precharge = 0; 4759 } 4760 /* 4761 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so 4762 * we must uncharge here. 4763 */ 4764 if (mc.moved_charge) { 4765 __mem_cgroup_cancel_charge(mc.from, mc.moved_charge); 4766 mc.moved_charge = 0; 4767 } 4768 /* we must fixup refcnts and charges */ 4769 if (mc.moved_swap) { 4770 /* uncharge swap account from the old cgroup */ 4771 if (!mem_cgroup_is_root(mc.from)) 4772 res_counter_uncharge(&mc.from->memsw, 4773 PAGE_SIZE * mc.moved_swap); 4774 __mem_cgroup_put(mc.from, mc.moved_swap); 4775 4776 if (!mem_cgroup_is_root(mc.to)) { 4777 /* 4778 * we charged both to->res and to->memsw, so we should 4779 * uncharge to->res. 4780 */ 4781 res_counter_uncharge(&mc.to->res, 4782 PAGE_SIZE * mc.moved_swap); 4783 } 4784 /* we've already done mem_cgroup_get(mc.to) */ 4785 mc.moved_swap = 0; 4786 } 4787 memcg_oom_recover(from); 4788 memcg_oom_recover(to); 4789 wake_up_all(&mc.waitq); 4790} 4791 4792static void mem_cgroup_clear_mc(void) 4793{ 4794 struct mem_cgroup *from = mc.from; 4795 4796 /* 4797 * we must clear moving_task before waking up waiters at the end of 4798 * task migration. 4799 */ 4800 mc.moving_task = NULL; 4801 __mem_cgroup_clear_mc(); 4802 spin_lock(&mc.lock); 4803 mc.from = NULL; 4804 mc.to = NULL; 4805 spin_unlock(&mc.lock); 4806 mem_cgroup_end_move(from); 4807} 4808 4809static int mem_cgroup_can_attach(struct cgroup_subsys *ss, 4810 struct cgroup *cgroup, 4811 struct task_struct *p, 4812 bool threadgroup) 4813{ 4814 int ret = 0; 4815 struct mem_cgroup *mem = mem_cgroup_from_cont(cgroup); 4816 4817 if (mem->move_charge_at_immigrate) { 4818 struct mm_struct *mm; 4819 struct mem_cgroup *from = mem_cgroup_from_task(p); 4820 4821 VM_BUG_ON(from == mem); 4822 4823 mm = get_task_mm(p); 4824 if (!mm) 4825 return 0; 4826 /* We move charges only when we move a owner of the mm */ 4827 if (mm->owner == p) { 4828 VM_BUG_ON(mc.from); 4829 VM_BUG_ON(mc.to); 4830 VM_BUG_ON(mc.precharge); 4831 VM_BUG_ON(mc.moved_charge); 4832 VM_BUG_ON(mc.moved_swap); 4833 mem_cgroup_start_move(from); 4834 spin_lock(&mc.lock); 4835 mc.from = from; 4836 mc.to = mem; 4837 spin_unlock(&mc.lock); 4838 /* We set mc.moving_task later */ 4839 4840 ret = mem_cgroup_precharge_mc(mm); 4841 if (ret) 4842 mem_cgroup_clear_mc(); 4843 } 4844 mmput(mm); 4845 } 4846 return ret; 4847} 4848 4849static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss, 4850 struct cgroup *cgroup, 4851 struct task_struct *p, 4852 bool threadgroup) 4853{ 4854 mem_cgroup_clear_mc(); 4855} 4856 4857static int mem_cgroup_move_charge_pte_range(pmd_t *pmd, 4858 unsigned long addr, unsigned long end, 4859 struct mm_walk *walk) 4860{ 4861 int ret = 0; 4862 struct vm_area_struct *vma = walk->private; 4863 pte_t *pte; 4864 spinlock_t *ptl; 4865 4866retry: 4867 VM_BUG_ON(pmd_trans_huge(*pmd)); 4868 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 4869 for (; addr != end; addr += PAGE_SIZE) { 4870 pte_t ptent = *(pte++); 4871 union mc_target target; 4872 int type; 4873 struct page *page; 4874 struct page_cgroup *pc; 4875 swp_entry_t ent; 4876 4877 if (!mc.precharge) 4878 break; 4879 4880 type = is_target_pte_for_mc(vma, addr, ptent, &target); 4881 switch (type) { 4882 case MC_TARGET_PAGE: 4883 page = target.page; 4884 if (isolate_lru_page(page)) 4885 goto put; 4886 pc = lookup_page_cgroup(page); 4887 if (!mem_cgroup_move_account(pc, 4888 mc.from, mc.to, false, PAGE_SIZE)) { 4889 mc.precharge--; 4890 /* we uncharge from mc.from later. */ 4891 mc.moved_charge++; 4892 } 4893 putback_lru_page(page); 4894put: /* is_target_pte_for_mc() gets the page */ 4895 put_page(page); 4896 break; 4897 case MC_TARGET_SWAP: 4898 ent = target.ent; 4899 if (!mem_cgroup_move_swap_account(ent, 4900 mc.from, mc.to, false)) { 4901 mc.precharge--; 4902 /* we fixup refcnts and charges later. */ 4903 mc.moved_swap++; 4904 } 4905 break; 4906 default: 4907 break; 4908 } 4909 } 4910 pte_unmap_unlock(pte - 1, ptl); 4911 cond_resched(); 4912 4913 if (addr != end) { 4914 /* 4915 * We have consumed all precharges we got in can_attach(). 4916 * We try charge one by one, but don't do any additional 4917 * charges to mc.to if we have failed in charge once in attach() 4918 * phase. 4919 */ 4920 ret = mem_cgroup_do_precharge(1); 4921 if (!ret) 4922 goto retry; 4923 } 4924 4925 return ret; 4926} 4927 4928static void mem_cgroup_move_charge(struct mm_struct *mm) 4929{ 4930 struct vm_area_struct *vma; 4931 4932 lru_add_drain_all(); 4933retry: 4934 if (unlikely(!down_read_trylock(&mm->mmap_sem))) { 4935 /* 4936 * Someone who are holding the mmap_sem might be waiting in 4937 * waitq. So we cancel all extra charges, wake up all waiters, 4938 * and retry. Because we cancel precharges, we might not be able 4939 * to move enough charges, but moving charge is a best-effort 4940 * feature anyway, so it wouldn't be a big problem. 4941 */ 4942 __mem_cgroup_clear_mc(); 4943 cond_resched(); 4944 goto retry; 4945 } 4946 for (vma = mm->mmap; vma; vma = vma->vm_next) { 4947 int ret; 4948 struct mm_walk mem_cgroup_move_charge_walk = { 4949 .pmd_entry = mem_cgroup_move_charge_pte_range, 4950 .mm = mm, 4951 .private = vma, 4952 }; 4953 if (is_vm_hugetlb_page(vma)) 4954 continue; 4955 ret = walk_page_range(vma->vm_start, vma->vm_end, 4956 &mem_cgroup_move_charge_walk); 4957 if (ret) 4958 /* 4959 * means we have consumed all precharges and failed in 4960 * doing additional charge. Just abandon here. 4961 */ 4962 break; 4963 } 4964 up_read(&mm->mmap_sem); 4965} 4966 4967static void mem_cgroup_move_task(struct cgroup_subsys *ss, 4968 struct cgroup *cont, 4969 struct cgroup *old_cont, 4970 struct task_struct *p, 4971 bool threadgroup) 4972{ 4973 struct mm_struct *mm; 4974 4975 if (!mc.to) 4976 /* no need to move charge */ 4977 return; 4978 4979 mm = get_task_mm(p); 4980 if (mm) { 4981 mem_cgroup_move_charge(mm); 4982 mmput(mm); 4983 } 4984 mem_cgroup_clear_mc(); 4985} 4986#else /* !CONFIG_MMU */ 4987static int mem_cgroup_can_attach(struct cgroup_subsys *ss, 4988 struct cgroup *cgroup, 4989 struct task_struct *p, 4990 bool threadgroup) 4991{ 4992 return 0; 4993} 4994static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss, 4995 struct cgroup *cgroup, 4996 struct task_struct *p, 4997 bool threadgroup) 4998{ 4999} 5000static void mem_cgroup_move_task(struct cgroup_subsys *ss, 5001 struct cgroup *cont, 5002 struct cgroup *old_cont, 5003 struct task_struct *p, 5004 bool threadgroup) 5005{ 5006} 5007#endif 5008 5009struct cgroup_subsys mem_cgroup_subsys = { 5010 .name = "memory", 5011 .subsys_id = mem_cgroup_subsys_id, 5012 .create = mem_cgroup_create, 5013 .pre_destroy = mem_cgroup_pre_destroy, 5014 .destroy = mem_cgroup_destroy, 5015 .populate = mem_cgroup_populate, 5016 .can_attach = mem_cgroup_can_attach, 5017 .cancel_attach = mem_cgroup_cancel_attach, 5018 .attach = mem_cgroup_move_task, 5019 .early_init = 0, 5020 .use_id = 1, 5021}; 5022 5023#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP 5024static int __init enable_swap_account(char *s) 5025{ 5026 /* consider enabled if no parameter or 1 is given */ 5027 if (!(*s) || !strcmp(s, "=1")) 5028 really_do_swap_account = 1; 5029 else if (!strcmp(s, "=0")) 5030 really_do_swap_account = 0; 5031 return 1; 5032} 5033__setup("swapaccount", enable_swap_account); 5034 5035static int __init disable_swap_account(char *s) 5036{ 5037 enable_swap_account("=0"); 5038 return 1; 5039} 5040__setup("noswapaccount", disable_swap_account); 5041#endif 5042