memcontrol.c revision ad324e94475a04cfcdfdb11ad20f8ea81268e411
1/* memcontrol.c - Memory Controller 2 * 3 * Copyright IBM Corporation, 2007 4 * Author Balbir Singh <balbir@linux.vnet.ibm.com> 5 * 6 * Copyright 2007 OpenVZ SWsoft Inc 7 * Author: Pavel Emelianov <xemul@openvz.org> 8 * 9 * Memory thresholds 10 * Copyright (C) 2009 Nokia Corporation 11 * Author: Kirill A. Shutemov 12 * 13 * This program is free software; you can redistribute it and/or modify 14 * it under the terms of the GNU General Public License as published by 15 * the Free Software Foundation; either version 2 of the License, or 16 * (at your option) any later version. 17 * 18 * This program is distributed in the hope that it will be useful, 19 * but WITHOUT ANY WARRANTY; without even the implied warranty of 20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 21 * GNU General Public License for more details. 22 */ 23 24#include <linux/res_counter.h> 25#include <linux/memcontrol.h> 26#include <linux/cgroup.h> 27#include <linux/mm.h> 28#include <linux/hugetlb.h> 29#include <linux/pagemap.h> 30#include <linux/smp.h> 31#include <linux/page-flags.h> 32#include <linux/backing-dev.h> 33#include <linux/bit_spinlock.h> 34#include <linux/rcupdate.h> 35#include <linux/limits.h> 36#include <linux/mutex.h> 37#include <linux/rbtree.h> 38#include <linux/slab.h> 39#include <linux/swap.h> 40#include <linux/swapops.h> 41#include <linux/spinlock.h> 42#include <linux/eventfd.h> 43#include <linux/sort.h> 44#include <linux/fs.h> 45#include <linux/seq_file.h> 46#include <linux/vmalloc.h> 47#include <linux/mm_inline.h> 48#include <linux/page_cgroup.h> 49#include <linux/cpu.h> 50#include <linux/oom.h> 51#include "internal.h" 52 53#include <asm/uaccess.h> 54 55#include <trace/events/vmscan.h> 56 57struct cgroup_subsys mem_cgroup_subsys __read_mostly; 58#define MEM_CGROUP_RECLAIM_RETRIES 5 59struct mem_cgroup *root_mem_cgroup __read_mostly; 60 61#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP 62/* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */ 63int do_swap_account __read_mostly; 64 65/* for remember boot option*/ 66#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP_ENABLED 67static int really_do_swap_account __initdata = 1; 68#else 69static int really_do_swap_account __initdata = 0; 70#endif 71 72#else 73#define do_swap_account (0) 74#endif 75 76/* 77 * Per memcg event counter is incremented at every pagein/pageout. This counter 78 * is used for trigger some periodic events. This is straightforward and better 79 * than using jiffies etc. to handle periodic memcg event. 80 * 81 * These values will be used as !((event) & ((1 <<(thresh)) - 1)) 82 */ 83#define THRESHOLDS_EVENTS_THRESH (7) /* once in 128 */ 84#define SOFTLIMIT_EVENTS_THRESH (10) /* once in 1024 */ 85 86/* 87 * Statistics for memory cgroup. 88 */ 89enum mem_cgroup_stat_index { 90 /* 91 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss. 92 */ 93 MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */ 94 MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */ 95 MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */ 96 MEM_CGROUP_STAT_PGPGIN_COUNT, /* # of pages paged in */ 97 MEM_CGROUP_STAT_PGPGOUT_COUNT, /* # of pages paged out */ 98 MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */ 99 MEM_CGROUP_STAT_DATA, /* end of data requires synchronization */ 100 /* incremented at every pagein/pageout */ 101 MEM_CGROUP_EVENTS = MEM_CGROUP_STAT_DATA, 102 MEM_CGROUP_ON_MOVE, /* someone is moving account between groups */ 103 104 MEM_CGROUP_STAT_NSTATS, 105}; 106 107struct mem_cgroup_stat_cpu { 108 s64 count[MEM_CGROUP_STAT_NSTATS]; 109}; 110 111/* 112 * per-zone information in memory controller. 113 */ 114struct mem_cgroup_per_zone { 115 /* 116 * spin_lock to protect the per cgroup LRU 117 */ 118 struct list_head lists[NR_LRU_LISTS]; 119 unsigned long count[NR_LRU_LISTS]; 120 121 struct zone_reclaim_stat reclaim_stat; 122 struct rb_node tree_node; /* RB tree node */ 123 unsigned long long usage_in_excess;/* Set to the value by which */ 124 /* the soft limit is exceeded*/ 125 bool on_tree; 126 struct mem_cgroup *mem; /* Back pointer, we cannot */ 127 /* use container_of */ 128}; 129/* Macro for accessing counter */ 130#define MEM_CGROUP_ZSTAT(mz, idx) ((mz)->count[(idx)]) 131 132struct mem_cgroup_per_node { 133 struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES]; 134}; 135 136struct mem_cgroup_lru_info { 137 struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES]; 138}; 139 140/* 141 * Cgroups above their limits are maintained in a RB-Tree, independent of 142 * their hierarchy representation 143 */ 144 145struct mem_cgroup_tree_per_zone { 146 struct rb_root rb_root; 147 spinlock_t lock; 148}; 149 150struct mem_cgroup_tree_per_node { 151 struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES]; 152}; 153 154struct mem_cgroup_tree { 155 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES]; 156}; 157 158static struct mem_cgroup_tree soft_limit_tree __read_mostly; 159 160struct mem_cgroup_threshold { 161 struct eventfd_ctx *eventfd; 162 u64 threshold; 163}; 164 165/* For threshold */ 166struct mem_cgroup_threshold_ary { 167 /* An array index points to threshold just below usage. */ 168 int current_threshold; 169 /* Size of entries[] */ 170 unsigned int size; 171 /* Array of thresholds */ 172 struct mem_cgroup_threshold entries[0]; 173}; 174 175struct mem_cgroup_thresholds { 176 /* Primary thresholds array */ 177 struct mem_cgroup_threshold_ary *primary; 178 /* 179 * Spare threshold array. 180 * This is needed to make mem_cgroup_unregister_event() "never fail". 181 * It must be able to store at least primary->size - 1 entries. 182 */ 183 struct mem_cgroup_threshold_ary *spare; 184}; 185 186/* for OOM */ 187struct mem_cgroup_eventfd_list { 188 struct list_head list; 189 struct eventfd_ctx *eventfd; 190}; 191 192static void mem_cgroup_threshold(struct mem_cgroup *mem); 193static void mem_cgroup_oom_notify(struct mem_cgroup *mem); 194 195/* 196 * The memory controller data structure. The memory controller controls both 197 * page cache and RSS per cgroup. We would eventually like to provide 198 * statistics based on the statistics developed by Rik Van Riel for clock-pro, 199 * to help the administrator determine what knobs to tune. 200 * 201 * TODO: Add a water mark for the memory controller. Reclaim will begin when 202 * we hit the water mark. May be even add a low water mark, such that 203 * no reclaim occurs from a cgroup at it's low water mark, this is 204 * a feature that will be implemented much later in the future. 205 */ 206struct mem_cgroup { 207 struct cgroup_subsys_state css; 208 /* 209 * the counter to account for memory usage 210 */ 211 struct res_counter res; 212 /* 213 * the counter to account for mem+swap usage. 214 */ 215 struct res_counter memsw; 216 /* 217 * Per cgroup active and inactive list, similar to the 218 * per zone LRU lists. 219 */ 220 struct mem_cgroup_lru_info info; 221 222 /* 223 protect against reclaim related member. 224 */ 225 spinlock_t reclaim_param_lock; 226 227 /* 228 * While reclaiming in a hierarchy, we cache the last child we 229 * reclaimed from. 230 */ 231 int last_scanned_child; 232 /* 233 * Should the accounting and control be hierarchical, per subtree? 234 */ 235 bool use_hierarchy; 236 atomic_t oom_lock; 237 atomic_t refcnt; 238 239 unsigned int swappiness; 240 /* OOM-Killer disable */ 241 int oom_kill_disable; 242 243 /* set when res.limit == memsw.limit */ 244 bool memsw_is_minimum; 245 246 /* protect arrays of thresholds */ 247 struct mutex thresholds_lock; 248 249 /* thresholds for memory usage. RCU-protected */ 250 struct mem_cgroup_thresholds thresholds; 251 252 /* thresholds for mem+swap usage. RCU-protected */ 253 struct mem_cgroup_thresholds memsw_thresholds; 254 255 /* For oom notifier event fd */ 256 struct list_head oom_notify; 257 258 /* 259 * Should we move charges of a task when a task is moved into this 260 * mem_cgroup ? And what type of charges should we move ? 261 */ 262 unsigned long move_charge_at_immigrate; 263 /* 264 * percpu counter. 265 */ 266 struct mem_cgroup_stat_cpu *stat; 267 /* 268 * used when a cpu is offlined or other synchronizations 269 * See mem_cgroup_read_stat(). 270 */ 271 struct mem_cgroup_stat_cpu nocpu_base; 272 spinlock_t pcp_counter_lock; 273}; 274 275/* Stuffs for move charges at task migration. */ 276/* 277 * Types of charges to be moved. "move_charge_at_immitgrate" is treated as a 278 * left-shifted bitmap of these types. 279 */ 280enum move_type { 281 MOVE_CHARGE_TYPE_ANON, /* private anonymous page and swap of it */ 282 MOVE_CHARGE_TYPE_FILE, /* file page(including tmpfs) and swap of it */ 283 NR_MOVE_TYPE, 284}; 285 286/* "mc" and its members are protected by cgroup_mutex */ 287static struct move_charge_struct { 288 spinlock_t lock; /* for from, to */ 289 struct mem_cgroup *from; 290 struct mem_cgroup *to; 291 unsigned long precharge; 292 unsigned long moved_charge; 293 unsigned long moved_swap; 294 struct task_struct *moving_task; /* a task moving charges */ 295 wait_queue_head_t waitq; /* a waitq for other context */ 296} mc = { 297 .lock = __SPIN_LOCK_UNLOCKED(mc.lock), 298 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq), 299}; 300 301static bool move_anon(void) 302{ 303 return test_bit(MOVE_CHARGE_TYPE_ANON, 304 &mc.to->move_charge_at_immigrate); 305} 306 307static bool move_file(void) 308{ 309 return test_bit(MOVE_CHARGE_TYPE_FILE, 310 &mc.to->move_charge_at_immigrate); 311} 312 313/* 314 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft 315 * limit reclaim to prevent infinite loops, if they ever occur. 316 */ 317#define MEM_CGROUP_MAX_RECLAIM_LOOPS (100) 318#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS (2) 319 320enum charge_type { 321 MEM_CGROUP_CHARGE_TYPE_CACHE = 0, 322 MEM_CGROUP_CHARGE_TYPE_MAPPED, 323 MEM_CGROUP_CHARGE_TYPE_SHMEM, /* used by page migration of shmem */ 324 MEM_CGROUP_CHARGE_TYPE_FORCE, /* used by force_empty */ 325 MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */ 326 MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */ 327 NR_CHARGE_TYPE, 328}; 329 330/* for encoding cft->private value on file */ 331#define _MEM (0) 332#define _MEMSWAP (1) 333#define _OOM_TYPE (2) 334#define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val)) 335#define MEMFILE_TYPE(val) (((val) >> 16) & 0xffff) 336#define MEMFILE_ATTR(val) ((val) & 0xffff) 337/* Used for OOM nofiier */ 338#define OOM_CONTROL (0) 339 340/* 341 * Reclaim flags for mem_cgroup_hierarchical_reclaim 342 */ 343#define MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0 344#define MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT) 345#define MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1 346#define MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT) 347#define MEM_CGROUP_RECLAIM_SOFT_BIT 0x2 348#define MEM_CGROUP_RECLAIM_SOFT (1 << MEM_CGROUP_RECLAIM_SOFT_BIT) 349 350static void mem_cgroup_get(struct mem_cgroup *mem); 351static void mem_cgroup_put(struct mem_cgroup *mem); 352static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem); 353static void drain_all_stock_async(void); 354 355static struct mem_cgroup_per_zone * 356mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid) 357{ 358 return &mem->info.nodeinfo[nid]->zoneinfo[zid]; 359} 360 361struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem) 362{ 363 return &mem->css; 364} 365 366static struct mem_cgroup_per_zone * 367page_cgroup_zoneinfo(struct page_cgroup *pc) 368{ 369 struct mem_cgroup *mem = pc->mem_cgroup; 370 int nid = page_cgroup_nid(pc); 371 int zid = page_cgroup_zid(pc); 372 373 return mem_cgroup_zoneinfo(mem, nid, zid); 374} 375 376static struct mem_cgroup_tree_per_zone * 377soft_limit_tree_node_zone(int nid, int zid) 378{ 379 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid]; 380} 381 382static struct mem_cgroup_tree_per_zone * 383soft_limit_tree_from_page(struct page *page) 384{ 385 int nid = page_to_nid(page); 386 int zid = page_zonenum(page); 387 388 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid]; 389} 390 391static void 392__mem_cgroup_insert_exceeded(struct mem_cgroup *mem, 393 struct mem_cgroup_per_zone *mz, 394 struct mem_cgroup_tree_per_zone *mctz, 395 unsigned long long new_usage_in_excess) 396{ 397 struct rb_node **p = &mctz->rb_root.rb_node; 398 struct rb_node *parent = NULL; 399 struct mem_cgroup_per_zone *mz_node; 400 401 if (mz->on_tree) 402 return; 403 404 mz->usage_in_excess = new_usage_in_excess; 405 if (!mz->usage_in_excess) 406 return; 407 while (*p) { 408 parent = *p; 409 mz_node = rb_entry(parent, struct mem_cgroup_per_zone, 410 tree_node); 411 if (mz->usage_in_excess < mz_node->usage_in_excess) 412 p = &(*p)->rb_left; 413 /* 414 * We can't avoid mem cgroups that are over their soft 415 * limit by the same amount 416 */ 417 else if (mz->usage_in_excess >= mz_node->usage_in_excess) 418 p = &(*p)->rb_right; 419 } 420 rb_link_node(&mz->tree_node, parent, p); 421 rb_insert_color(&mz->tree_node, &mctz->rb_root); 422 mz->on_tree = true; 423} 424 425static void 426__mem_cgroup_remove_exceeded(struct mem_cgroup *mem, 427 struct mem_cgroup_per_zone *mz, 428 struct mem_cgroup_tree_per_zone *mctz) 429{ 430 if (!mz->on_tree) 431 return; 432 rb_erase(&mz->tree_node, &mctz->rb_root); 433 mz->on_tree = false; 434} 435 436static void 437mem_cgroup_remove_exceeded(struct mem_cgroup *mem, 438 struct mem_cgroup_per_zone *mz, 439 struct mem_cgroup_tree_per_zone *mctz) 440{ 441 spin_lock(&mctz->lock); 442 __mem_cgroup_remove_exceeded(mem, mz, mctz); 443 spin_unlock(&mctz->lock); 444} 445 446 447static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page) 448{ 449 unsigned long long excess; 450 struct mem_cgroup_per_zone *mz; 451 struct mem_cgroup_tree_per_zone *mctz; 452 int nid = page_to_nid(page); 453 int zid = page_zonenum(page); 454 mctz = soft_limit_tree_from_page(page); 455 456 /* 457 * Necessary to update all ancestors when hierarchy is used. 458 * because their event counter is not touched. 459 */ 460 for (; mem; mem = parent_mem_cgroup(mem)) { 461 mz = mem_cgroup_zoneinfo(mem, nid, zid); 462 excess = res_counter_soft_limit_excess(&mem->res); 463 /* 464 * We have to update the tree if mz is on RB-tree or 465 * mem is over its softlimit. 466 */ 467 if (excess || mz->on_tree) { 468 spin_lock(&mctz->lock); 469 /* if on-tree, remove it */ 470 if (mz->on_tree) 471 __mem_cgroup_remove_exceeded(mem, mz, mctz); 472 /* 473 * Insert again. mz->usage_in_excess will be updated. 474 * If excess is 0, no tree ops. 475 */ 476 __mem_cgroup_insert_exceeded(mem, mz, mctz, excess); 477 spin_unlock(&mctz->lock); 478 } 479 } 480} 481 482static void mem_cgroup_remove_from_trees(struct mem_cgroup *mem) 483{ 484 int node, zone; 485 struct mem_cgroup_per_zone *mz; 486 struct mem_cgroup_tree_per_zone *mctz; 487 488 for_each_node_state(node, N_POSSIBLE) { 489 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 490 mz = mem_cgroup_zoneinfo(mem, node, zone); 491 mctz = soft_limit_tree_node_zone(node, zone); 492 mem_cgroup_remove_exceeded(mem, mz, mctz); 493 } 494 } 495} 496 497static struct mem_cgroup_per_zone * 498__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz) 499{ 500 struct rb_node *rightmost = NULL; 501 struct mem_cgroup_per_zone *mz; 502 503retry: 504 mz = NULL; 505 rightmost = rb_last(&mctz->rb_root); 506 if (!rightmost) 507 goto done; /* Nothing to reclaim from */ 508 509 mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node); 510 /* 511 * Remove the node now but someone else can add it back, 512 * we will to add it back at the end of reclaim to its correct 513 * position in the tree. 514 */ 515 __mem_cgroup_remove_exceeded(mz->mem, mz, mctz); 516 if (!res_counter_soft_limit_excess(&mz->mem->res) || 517 !css_tryget(&mz->mem->css)) 518 goto retry; 519done: 520 return mz; 521} 522 523static struct mem_cgroup_per_zone * 524mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz) 525{ 526 struct mem_cgroup_per_zone *mz; 527 528 spin_lock(&mctz->lock); 529 mz = __mem_cgroup_largest_soft_limit_node(mctz); 530 spin_unlock(&mctz->lock); 531 return mz; 532} 533 534/* 535 * Implementation Note: reading percpu statistics for memcg. 536 * 537 * Both of vmstat[] and percpu_counter has threshold and do periodic 538 * synchronization to implement "quick" read. There are trade-off between 539 * reading cost and precision of value. Then, we may have a chance to implement 540 * a periodic synchronizion of counter in memcg's counter. 541 * 542 * But this _read() function is used for user interface now. The user accounts 543 * memory usage by memory cgroup and he _always_ requires exact value because 544 * he accounts memory. Even if we provide quick-and-fuzzy read, we always 545 * have to visit all online cpus and make sum. So, for now, unnecessary 546 * synchronization is not implemented. (just implemented for cpu hotplug) 547 * 548 * If there are kernel internal actions which can make use of some not-exact 549 * value, and reading all cpu value can be performance bottleneck in some 550 * common workload, threashold and synchonization as vmstat[] should be 551 * implemented. 552 */ 553static s64 mem_cgroup_read_stat(struct mem_cgroup *mem, 554 enum mem_cgroup_stat_index idx) 555{ 556 int cpu; 557 s64 val = 0; 558 559 get_online_cpus(); 560 for_each_online_cpu(cpu) 561 val += per_cpu(mem->stat->count[idx], cpu); 562#ifdef CONFIG_HOTPLUG_CPU 563 spin_lock(&mem->pcp_counter_lock); 564 val += mem->nocpu_base.count[idx]; 565 spin_unlock(&mem->pcp_counter_lock); 566#endif 567 put_online_cpus(); 568 return val; 569} 570 571static s64 mem_cgroup_local_usage(struct mem_cgroup *mem) 572{ 573 s64 ret; 574 575 ret = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS); 576 ret += mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE); 577 return ret; 578} 579 580static void mem_cgroup_swap_statistics(struct mem_cgroup *mem, 581 bool charge) 582{ 583 int val = (charge) ? 1 : -1; 584 this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_SWAPOUT], val); 585} 586 587static void mem_cgroup_charge_statistics(struct mem_cgroup *mem, 588 bool file, int nr_pages) 589{ 590 preempt_disable(); 591 592 if (file) 593 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_CACHE], nr_pages); 594 else 595 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_RSS], nr_pages); 596 597 /* pagein of a big page is an event. So, ignore page size */ 598 if (nr_pages > 0) 599 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGIN_COUNT]); 600 else { 601 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGOUT_COUNT]); 602 nr_pages = -nr_pages; /* for event */ 603 } 604 605 __this_cpu_add(mem->stat->count[MEM_CGROUP_EVENTS], nr_pages); 606 607 preempt_enable(); 608} 609 610static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem, 611 enum lru_list idx) 612{ 613 int nid, zid; 614 struct mem_cgroup_per_zone *mz; 615 u64 total = 0; 616 617 for_each_online_node(nid) 618 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 619 mz = mem_cgroup_zoneinfo(mem, nid, zid); 620 total += MEM_CGROUP_ZSTAT(mz, idx); 621 } 622 return total; 623} 624 625static bool __memcg_event_check(struct mem_cgroup *mem, int event_mask_shift) 626{ 627 s64 val; 628 629 val = this_cpu_read(mem->stat->count[MEM_CGROUP_EVENTS]); 630 631 return !(val & ((1 << event_mask_shift) - 1)); 632} 633 634/* 635 * Check events in order. 636 * 637 */ 638static void memcg_check_events(struct mem_cgroup *mem, struct page *page) 639{ 640 /* threshold event is triggered in finer grain than soft limit */ 641 if (unlikely(__memcg_event_check(mem, THRESHOLDS_EVENTS_THRESH))) { 642 mem_cgroup_threshold(mem); 643 if (unlikely(__memcg_event_check(mem, SOFTLIMIT_EVENTS_THRESH))) 644 mem_cgroup_update_tree(mem, page); 645 } 646} 647 648static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont) 649{ 650 return container_of(cgroup_subsys_state(cont, 651 mem_cgroup_subsys_id), struct mem_cgroup, 652 css); 653} 654 655struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) 656{ 657 /* 658 * mm_update_next_owner() may clear mm->owner to NULL 659 * if it races with swapoff, page migration, etc. 660 * So this can be called with p == NULL. 661 */ 662 if (unlikely(!p)) 663 return NULL; 664 665 return container_of(task_subsys_state(p, mem_cgroup_subsys_id), 666 struct mem_cgroup, css); 667} 668 669static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm) 670{ 671 struct mem_cgroup *mem = NULL; 672 673 if (!mm) 674 return NULL; 675 /* 676 * Because we have no locks, mm->owner's may be being moved to other 677 * cgroup. We use css_tryget() here even if this looks 678 * pessimistic (rather than adding locks here). 679 */ 680 rcu_read_lock(); 681 do { 682 mem = mem_cgroup_from_task(rcu_dereference(mm->owner)); 683 if (unlikely(!mem)) 684 break; 685 } while (!css_tryget(&mem->css)); 686 rcu_read_unlock(); 687 return mem; 688} 689 690/* The caller has to guarantee "mem" exists before calling this */ 691static struct mem_cgroup *mem_cgroup_start_loop(struct mem_cgroup *mem) 692{ 693 struct cgroup_subsys_state *css; 694 int found; 695 696 if (!mem) /* ROOT cgroup has the smallest ID */ 697 return root_mem_cgroup; /*css_put/get against root is ignored*/ 698 if (!mem->use_hierarchy) { 699 if (css_tryget(&mem->css)) 700 return mem; 701 return NULL; 702 } 703 rcu_read_lock(); 704 /* 705 * searching a memory cgroup which has the smallest ID under given 706 * ROOT cgroup. (ID >= 1) 707 */ 708 css = css_get_next(&mem_cgroup_subsys, 1, &mem->css, &found); 709 if (css && css_tryget(css)) 710 mem = container_of(css, struct mem_cgroup, css); 711 else 712 mem = NULL; 713 rcu_read_unlock(); 714 return mem; 715} 716 717static struct mem_cgroup *mem_cgroup_get_next(struct mem_cgroup *iter, 718 struct mem_cgroup *root, 719 bool cond) 720{ 721 int nextid = css_id(&iter->css) + 1; 722 int found; 723 int hierarchy_used; 724 struct cgroup_subsys_state *css; 725 726 hierarchy_used = iter->use_hierarchy; 727 728 css_put(&iter->css); 729 /* If no ROOT, walk all, ignore hierarchy */ 730 if (!cond || (root && !hierarchy_used)) 731 return NULL; 732 733 if (!root) 734 root = root_mem_cgroup; 735 736 do { 737 iter = NULL; 738 rcu_read_lock(); 739 740 css = css_get_next(&mem_cgroup_subsys, nextid, 741 &root->css, &found); 742 if (css && css_tryget(css)) 743 iter = container_of(css, struct mem_cgroup, css); 744 rcu_read_unlock(); 745 /* If css is NULL, no more cgroups will be found */ 746 nextid = found + 1; 747 } while (css && !iter); 748 749 return iter; 750} 751/* 752 * for_eacn_mem_cgroup_tree() for visiting all cgroup under tree. Please 753 * be careful that "break" loop is not allowed. We have reference count. 754 * Instead of that modify "cond" to be false and "continue" to exit the loop. 755 */ 756#define for_each_mem_cgroup_tree_cond(iter, root, cond) \ 757 for (iter = mem_cgroup_start_loop(root);\ 758 iter != NULL;\ 759 iter = mem_cgroup_get_next(iter, root, cond)) 760 761#define for_each_mem_cgroup_tree(iter, root) \ 762 for_each_mem_cgroup_tree_cond(iter, root, true) 763 764#define for_each_mem_cgroup_all(iter) \ 765 for_each_mem_cgroup_tree_cond(iter, NULL, true) 766 767 768static inline bool mem_cgroup_is_root(struct mem_cgroup *mem) 769{ 770 return (mem == root_mem_cgroup); 771} 772 773/* 774 * Following LRU functions are allowed to be used without PCG_LOCK. 775 * Operations are called by routine of global LRU independently from memcg. 776 * What we have to take care of here is validness of pc->mem_cgroup. 777 * 778 * Changes to pc->mem_cgroup happens when 779 * 1. charge 780 * 2. moving account 781 * In typical case, "charge" is done before add-to-lru. Exception is SwapCache. 782 * It is added to LRU before charge. 783 * If PCG_USED bit is not set, page_cgroup is not added to this private LRU. 784 * When moving account, the page is not on LRU. It's isolated. 785 */ 786 787void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru) 788{ 789 struct page_cgroup *pc; 790 struct mem_cgroup_per_zone *mz; 791 792 if (mem_cgroup_disabled()) 793 return; 794 pc = lookup_page_cgroup(page); 795 /* can happen while we handle swapcache. */ 796 if (!TestClearPageCgroupAcctLRU(pc)) 797 return; 798 VM_BUG_ON(!pc->mem_cgroup); 799 /* 800 * We don't check PCG_USED bit. It's cleared when the "page" is finally 801 * removed from global LRU. 802 */ 803 mz = page_cgroup_zoneinfo(pc); 804 /* huge page split is done under lru_lock. so, we have no races. */ 805 MEM_CGROUP_ZSTAT(mz, lru) -= 1 << compound_order(page); 806 if (mem_cgroup_is_root(pc->mem_cgroup)) 807 return; 808 VM_BUG_ON(list_empty(&pc->lru)); 809 list_del_init(&pc->lru); 810} 811 812void mem_cgroup_del_lru(struct page *page) 813{ 814 mem_cgroup_del_lru_list(page, page_lru(page)); 815} 816 817/* 818 * Writeback is about to end against a page which has been marked for immediate 819 * reclaim. If it still appears to be reclaimable, move it to the tail of the 820 * inactive list. 821 */ 822void mem_cgroup_rotate_reclaimable_page(struct page *page) 823{ 824 struct mem_cgroup_per_zone *mz; 825 struct page_cgroup *pc; 826 enum lru_list lru = page_lru(page); 827 828 if (mem_cgroup_disabled()) 829 return; 830 831 pc = lookup_page_cgroup(page); 832 /* unused or root page is not rotated. */ 833 if (!PageCgroupUsed(pc)) 834 return; 835 /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */ 836 smp_rmb(); 837 if (mem_cgroup_is_root(pc->mem_cgroup)) 838 return; 839 mz = page_cgroup_zoneinfo(pc); 840 list_move_tail(&pc->lru, &mz->lists[lru]); 841} 842 843void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru) 844{ 845 struct mem_cgroup_per_zone *mz; 846 struct page_cgroup *pc; 847 848 if (mem_cgroup_disabled()) 849 return; 850 851 pc = lookup_page_cgroup(page); 852 /* unused or root page is not rotated. */ 853 if (!PageCgroupUsed(pc)) 854 return; 855 /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */ 856 smp_rmb(); 857 if (mem_cgroup_is_root(pc->mem_cgroup)) 858 return; 859 mz = page_cgroup_zoneinfo(pc); 860 list_move(&pc->lru, &mz->lists[lru]); 861} 862 863void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru) 864{ 865 struct page_cgroup *pc; 866 struct mem_cgroup_per_zone *mz; 867 868 if (mem_cgroup_disabled()) 869 return; 870 pc = lookup_page_cgroup(page); 871 VM_BUG_ON(PageCgroupAcctLRU(pc)); 872 if (!PageCgroupUsed(pc)) 873 return; 874 /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */ 875 smp_rmb(); 876 mz = page_cgroup_zoneinfo(pc); 877 /* huge page split is done under lru_lock. so, we have no races. */ 878 MEM_CGROUP_ZSTAT(mz, lru) += 1 << compound_order(page); 879 SetPageCgroupAcctLRU(pc); 880 if (mem_cgroup_is_root(pc->mem_cgroup)) 881 return; 882 list_add(&pc->lru, &mz->lists[lru]); 883} 884 885/* 886 * At handling SwapCache, pc->mem_cgroup may be changed while it's linked to 887 * lru because the page may.be reused after it's fully uncharged (because of 888 * SwapCache behavior).To handle that, unlink page_cgroup from LRU when charge 889 * it again. This function is only used to charge SwapCache. It's done under 890 * lock_page and expected that zone->lru_lock is never held. 891 */ 892static void mem_cgroup_lru_del_before_commit_swapcache(struct page *page) 893{ 894 unsigned long flags; 895 struct zone *zone = page_zone(page); 896 struct page_cgroup *pc = lookup_page_cgroup(page); 897 898 spin_lock_irqsave(&zone->lru_lock, flags); 899 /* 900 * Forget old LRU when this page_cgroup is *not* used. This Used bit 901 * is guarded by lock_page() because the page is SwapCache. 902 */ 903 if (!PageCgroupUsed(pc)) 904 mem_cgroup_del_lru_list(page, page_lru(page)); 905 spin_unlock_irqrestore(&zone->lru_lock, flags); 906} 907 908static void mem_cgroup_lru_add_after_commit_swapcache(struct page *page) 909{ 910 unsigned long flags; 911 struct zone *zone = page_zone(page); 912 struct page_cgroup *pc = lookup_page_cgroup(page); 913 914 spin_lock_irqsave(&zone->lru_lock, flags); 915 /* link when the page is linked to LRU but page_cgroup isn't */ 916 if (PageLRU(page) && !PageCgroupAcctLRU(pc)) 917 mem_cgroup_add_lru_list(page, page_lru(page)); 918 spin_unlock_irqrestore(&zone->lru_lock, flags); 919} 920 921 922void mem_cgroup_move_lists(struct page *page, 923 enum lru_list from, enum lru_list to) 924{ 925 if (mem_cgroup_disabled()) 926 return; 927 mem_cgroup_del_lru_list(page, from); 928 mem_cgroup_add_lru_list(page, to); 929} 930 931int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem) 932{ 933 int ret; 934 struct mem_cgroup *curr = NULL; 935 struct task_struct *p; 936 937 p = find_lock_task_mm(task); 938 if (!p) 939 return 0; 940 curr = try_get_mem_cgroup_from_mm(p->mm); 941 task_unlock(p); 942 if (!curr) 943 return 0; 944 /* 945 * We should check use_hierarchy of "mem" not "curr". Because checking 946 * use_hierarchy of "curr" here make this function true if hierarchy is 947 * enabled in "curr" and "curr" is a child of "mem" in *cgroup* 948 * hierarchy(even if use_hierarchy is disabled in "mem"). 949 */ 950 if (mem->use_hierarchy) 951 ret = css_is_ancestor(&curr->css, &mem->css); 952 else 953 ret = (curr == mem); 954 css_put(&curr->css); 955 return ret; 956} 957 958static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_pages) 959{ 960 unsigned long active; 961 unsigned long inactive; 962 unsigned long gb; 963 unsigned long inactive_ratio; 964 965 inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON); 966 active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON); 967 968 gb = (inactive + active) >> (30 - PAGE_SHIFT); 969 if (gb) 970 inactive_ratio = int_sqrt(10 * gb); 971 else 972 inactive_ratio = 1; 973 974 if (present_pages) { 975 present_pages[0] = inactive; 976 present_pages[1] = active; 977 } 978 979 return inactive_ratio; 980} 981 982int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg) 983{ 984 unsigned long active; 985 unsigned long inactive; 986 unsigned long present_pages[2]; 987 unsigned long inactive_ratio; 988 989 inactive_ratio = calc_inactive_ratio(memcg, present_pages); 990 991 inactive = present_pages[0]; 992 active = present_pages[1]; 993 994 if (inactive * inactive_ratio < active) 995 return 1; 996 997 return 0; 998} 999 1000int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg) 1001{ 1002 unsigned long active; 1003 unsigned long inactive; 1004 1005 inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE); 1006 active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE); 1007 1008 return (active > inactive); 1009} 1010 1011unsigned long mem_cgroup_zone_nr_pages(struct mem_cgroup *memcg, 1012 struct zone *zone, 1013 enum lru_list lru) 1014{ 1015 int nid = zone_to_nid(zone); 1016 int zid = zone_idx(zone); 1017 struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid); 1018 1019 return MEM_CGROUP_ZSTAT(mz, lru); 1020} 1021 1022struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg, 1023 struct zone *zone) 1024{ 1025 int nid = zone_to_nid(zone); 1026 int zid = zone_idx(zone); 1027 struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid); 1028 1029 return &mz->reclaim_stat; 1030} 1031 1032struct zone_reclaim_stat * 1033mem_cgroup_get_reclaim_stat_from_page(struct page *page) 1034{ 1035 struct page_cgroup *pc; 1036 struct mem_cgroup_per_zone *mz; 1037 1038 if (mem_cgroup_disabled()) 1039 return NULL; 1040 1041 pc = lookup_page_cgroup(page); 1042 if (!PageCgroupUsed(pc)) 1043 return NULL; 1044 /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */ 1045 smp_rmb(); 1046 mz = page_cgroup_zoneinfo(pc); 1047 if (!mz) 1048 return NULL; 1049 1050 return &mz->reclaim_stat; 1051} 1052 1053unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan, 1054 struct list_head *dst, 1055 unsigned long *scanned, int order, 1056 int mode, struct zone *z, 1057 struct mem_cgroup *mem_cont, 1058 int active, int file) 1059{ 1060 unsigned long nr_taken = 0; 1061 struct page *page; 1062 unsigned long scan; 1063 LIST_HEAD(pc_list); 1064 struct list_head *src; 1065 struct page_cgroup *pc, *tmp; 1066 int nid = zone_to_nid(z); 1067 int zid = zone_idx(z); 1068 struct mem_cgroup_per_zone *mz; 1069 int lru = LRU_FILE * file + active; 1070 int ret; 1071 1072 BUG_ON(!mem_cont); 1073 mz = mem_cgroup_zoneinfo(mem_cont, nid, zid); 1074 src = &mz->lists[lru]; 1075 1076 scan = 0; 1077 list_for_each_entry_safe_reverse(pc, tmp, src, lru) { 1078 if (scan >= nr_to_scan) 1079 break; 1080 1081 page = pc->page; 1082 if (unlikely(!PageCgroupUsed(pc))) 1083 continue; 1084 if (unlikely(!PageLRU(page))) 1085 continue; 1086 1087 scan++; 1088 ret = __isolate_lru_page(page, mode, file); 1089 switch (ret) { 1090 case 0: 1091 list_move(&page->lru, dst); 1092 mem_cgroup_del_lru(page); 1093 nr_taken += hpage_nr_pages(page); 1094 break; 1095 case -EBUSY: 1096 /* we don't affect global LRU but rotate in our LRU */ 1097 mem_cgroup_rotate_lru_list(page, page_lru(page)); 1098 break; 1099 default: 1100 break; 1101 } 1102 } 1103 1104 *scanned = scan; 1105 1106 trace_mm_vmscan_memcg_isolate(0, nr_to_scan, scan, nr_taken, 1107 0, 0, 0, mode); 1108 1109 return nr_taken; 1110} 1111 1112#define mem_cgroup_from_res_counter(counter, member) \ 1113 container_of(counter, struct mem_cgroup, member) 1114 1115/** 1116 * mem_cgroup_margin - calculate chargeable space of a memory cgroup 1117 * @mem: the memory cgroup 1118 * 1119 * Returns the maximum amount of memory @mem can be charged with, in 1120 * bytes. 1121 */ 1122static unsigned long long mem_cgroup_margin(struct mem_cgroup *mem) 1123{ 1124 unsigned long long margin; 1125 1126 margin = res_counter_margin(&mem->res); 1127 if (do_swap_account) 1128 margin = min(margin, res_counter_margin(&mem->memsw)); 1129 return margin; 1130} 1131 1132static unsigned int get_swappiness(struct mem_cgroup *memcg) 1133{ 1134 struct cgroup *cgrp = memcg->css.cgroup; 1135 unsigned int swappiness; 1136 1137 /* root ? */ 1138 if (cgrp->parent == NULL) 1139 return vm_swappiness; 1140 1141 spin_lock(&memcg->reclaim_param_lock); 1142 swappiness = memcg->swappiness; 1143 spin_unlock(&memcg->reclaim_param_lock); 1144 1145 return swappiness; 1146} 1147 1148static void mem_cgroup_start_move(struct mem_cgroup *mem) 1149{ 1150 int cpu; 1151 1152 get_online_cpus(); 1153 spin_lock(&mem->pcp_counter_lock); 1154 for_each_online_cpu(cpu) 1155 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) += 1; 1156 mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] += 1; 1157 spin_unlock(&mem->pcp_counter_lock); 1158 put_online_cpus(); 1159 1160 synchronize_rcu(); 1161} 1162 1163static void mem_cgroup_end_move(struct mem_cgroup *mem) 1164{ 1165 int cpu; 1166 1167 if (!mem) 1168 return; 1169 get_online_cpus(); 1170 spin_lock(&mem->pcp_counter_lock); 1171 for_each_online_cpu(cpu) 1172 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) -= 1; 1173 mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] -= 1; 1174 spin_unlock(&mem->pcp_counter_lock); 1175 put_online_cpus(); 1176} 1177/* 1178 * 2 routines for checking "mem" is under move_account() or not. 1179 * 1180 * mem_cgroup_stealed() - checking a cgroup is mc.from or not. This is used 1181 * for avoiding race in accounting. If true, 1182 * pc->mem_cgroup may be overwritten. 1183 * 1184 * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or 1185 * under hierarchy of moving cgroups. This is for 1186 * waiting at hith-memory prressure caused by "move". 1187 */ 1188 1189static bool mem_cgroup_stealed(struct mem_cgroup *mem) 1190{ 1191 VM_BUG_ON(!rcu_read_lock_held()); 1192 return this_cpu_read(mem->stat->count[MEM_CGROUP_ON_MOVE]) > 0; 1193} 1194 1195static bool mem_cgroup_under_move(struct mem_cgroup *mem) 1196{ 1197 struct mem_cgroup *from; 1198 struct mem_cgroup *to; 1199 bool ret = false; 1200 /* 1201 * Unlike task_move routines, we access mc.to, mc.from not under 1202 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead. 1203 */ 1204 spin_lock(&mc.lock); 1205 from = mc.from; 1206 to = mc.to; 1207 if (!from) 1208 goto unlock; 1209 if (from == mem || to == mem 1210 || (mem->use_hierarchy && css_is_ancestor(&from->css, &mem->css)) 1211 || (mem->use_hierarchy && css_is_ancestor(&to->css, &mem->css))) 1212 ret = true; 1213unlock: 1214 spin_unlock(&mc.lock); 1215 return ret; 1216} 1217 1218static bool mem_cgroup_wait_acct_move(struct mem_cgroup *mem) 1219{ 1220 if (mc.moving_task && current != mc.moving_task) { 1221 if (mem_cgroup_under_move(mem)) { 1222 DEFINE_WAIT(wait); 1223 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE); 1224 /* moving charge context might have finished. */ 1225 if (mc.moving_task) 1226 schedule(); 1227 finish_wait(&mc.waitq, &wait); 1228 return true; 1229 } 1230 } 1231 return false; 1232} 1233 1234/** 1235 * mem_cgroup_print_oom_info: Called from OOM with tasklist_lock held in read mode. 1236 * @memcg: The memory cgroup that went over limit 1237 * @p: Task that is going to be killed 1238 * 1239 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is 1240 * enabled 1241 */ 1242void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p) 1243{ 1244 struct cgroup *task_cgrp; 1245 struct cgroup *mem_cgrp; 1246 /* 1247 * Need a buffer in BSS, can't rely on allocations. The code relies 1248 * on the assumption that OOM is serialized for memory controller. 1249 * If this assumption is broken, revisit this code. 1250 */ 1251 static char memcg_name[PATH_MAX]; 1252 int ret; 1253 1254 if (!memcg || !p) 1255 return; 1256 1257 1258 rcu_read_lock(); 1259 1260 mem_cgrp = memcg->css.cgroup; 1261 task_cgrp = task_cgroup(p, mem_cgroup_subsys_id); 1262 1263 ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX); 1264 if (ret < 0) { 1265 /* 1266 * Unfortunately, we are unable to convert to a useful name 1267 * But we'll still print out the usage information 1268 */ 1269 rcu_read_unlock(); 1270 goto done; 1271 } 1272 rcu_read_unlock(); 1273 1274 printk(KERN_INFO "Task in %s killed", memcg_name); 1275 1276 rcu_read_lock(); 1277 ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX); 1278 if (ret < 0) { 1279 rcu_read_unlock(); 1280 goto done; 1281 } 1282 rcu_read_unlock(); 1283 1284 /* 1285 * Continues from above, so we don't need an KERN_ level 1286 */ 1287 printk(KERN_CONT " as a result of limit of %s\n", memcg_name); 1288done: 1289 1290 printk(KERN_INFO "memory: usage %llukB, limit %llukB, failcnt %llu\n", 1291 res_counter_read_u64(&memcg->res, RES_USAGE) >> 10, 1292 res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10, 1293 res_counter_read_u64(&memcg->res, RES_FAILCNT)); 1294 printk(KERN_INFO "memory+swap: usage %llukB, limit %llukB, " 1295 "failcnt %llu\n", 1296 res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10, 1297 res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10, 1298 res_counter_read_u64(&memcg->memsw, RES_FAILCNT)); 1299} 1300 1301/* 1302 * This function returns the number of memcg under hierarchy tree. Returns 1303 * 1(self count) if no children. 1304 */ 1305static int mem_cgroup_count_children(struct mem_cgroup *mem) 1306{ 1307 int num = 0; 1308 struct mem_cgroup *iter; 1309 1310 for_each_mem_cgroup_tree(iter, mem) 1311 num++; 1312 return num; 1313} 1314 1315/* 1316 * Return the memory (and swap, if configured) limit for a memcg. 1317 */ 1318u64 mem_cgroup_get_limit(struct mem_cgroup *memcg) 1319{ 1320 u64 limit; 1321 u64 memsw; 1322 1323 limit = res_counter_read_u64(&memcg->res, RES_LIMIT); 1324 limit += total_swap_pages << PAGE_SHIFT; 1325 1326 memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 1327 /* 1328 * If memsw is finite and limits the amount of swap space available 1329 * to this memcg, return that limit. 1330 */ 1331 return min(limit, memsw); 1332} 1333 1334/* 1335 * Visit the first child (need not be the first child as per the ordering 1336 * of the cgroup list, since we track last_scanned_child) of @mem and use 1337 * that to reclaim free pages from. 1338 */ 1339static struct mem_cgroup * 1340mem_cgroup_select_victim(struct mem_cgroup *root_mem) 1341{ 1342 struct mem_cgroup *ret = NULL; 1343 struct cgroup_subsys_state *css; 1344 int nextid, found; 1345 1346 if (!root_mem->use_hierarchy) { 1347 css_get(&root_mem->css); 1348 ret = root_mem; 1349 } 1350 1351 while (!ret) { 1352 rcu_read_lock(); 1353 nextid = root_mem->last_scanned_child + 1; 1354 css = css_get_next(&mem_cgroup_subsys, nextid, &root_mem->css, 1355 &found); 1356 if (css && css_tryget(css)) 1357 ret = container_of(css, struct mem_cgroup, css); 1358 1359 rcu_read_unlock(); 1360 /* Updates scanning parameter */ 1361 spin_lock(&root_mem->reclaim_param_lock); 1362 if (!css) { 1363 /* this means start scan from ID:1 */ 1364 root_mem->last_scanned_child = 0; 1365 } else 1366 root_mem->last_scanned_child = found; 1367 spin_unlock(&root_mem->reclaim_param_lock); 1368 } 1369 1370 return ret; 1371} 1372 1373/* 1374 * Scan the hierarchy if needed to reclaim memory. We remember the last child 1375 * we reclaimed from, so that we don't end up penalizing one child extensively 1376 * based on its position in the children list. 1377 * 1378 * root_mem is the original ancestor that we've been reclaim from. 1379 * 1380 * We give up and return to the caller when we visit root_mem twice. 1381 * (other groups can be removed while we're walking....) 1382 * 1383 * If shrink==true, for avoiding to free too much, this returns immedieately. 1384 */ 1385static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem, 1386 struct zone *zone, 1387 gfp_t gfp_mask, 1388 unsigned long reclaim_options) 1389{ 1390 struct mem_cgroup *victim; 1391 int ret, total = 0; 1392 int loop = 0; 1393 bool noswap = reclaim_options & MEM_CGROUP_RECLAIM_NOSWAP; 1394 bool shrink = reclaim_options & MEM_CGROUP_RECLAIM_SHRINK; 1395 bool check_soft = reclaim_options & MEM_CGROUP_RECLAIM_SOFT; 1396 unsigned long excess; 1397 1398 excess = res_counter_soft_limit_excess(&root_mem->res) >> PAGE_SHIFT; 1399 1400 /* If memsw_is_minimum==1, swap-out is of-no-use. */ 1401 if (root_mem->memsw_is_minimum) 1402 noswap = true; 1403 1404 while (1) { 1405 victim = mem_cgroup_select_victim(root_mem); 1406 if (victim == root_mem) { 1407 loop++; 1408 if (loop >= 1) 1409 drain_all_stock_async(); 1410 if (loop >= 2) { 1411 /* 1412 * If we have not been able to reclaim 1413 * anything, it might because there are 1414 * no reclaimable pages under this hierarchy 1415 */ 1416 if (!check_soft || !total) { 1417 css_put(&victim->css); 1418 break; 1419 } 1420 /* 1421 * We want to do more targetted reclaim. 1422 * excess >> 2 is not to excessive so as to 1423 * reclaim too much, nor too less that we keep 1424 * coming back to reclaim from this cgroup 1425 */ 1426 if (total >= (excess >> 2) || 1427 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) { 1428 css_put(&victim->css); 1429 break; 1430 } 1431 } 1432 } 1433 if (!mem_cgroup_local_usage(victim)) { 1434 /* this cgroup's local usage == 0 */ 1435 css_put(&victim->css); 1436 continue; 1437 } 1438 /* we use swappiness of local cgroup */ 1439 if (check_soft) 1440 ret = mem_cgroup_shrink_node_zone(victim, gfp_mask, 1441 noswap, get_swappiness(victim), zone); 1442 else 1443 ret = try_to_free_mem_cgroup_pages(victim, gfp_mask, 1444 noswap, get_swappiness(victim)); 1445 css_put(&victim->css); 1446 /* 1447 * At shrinking usage, we can't check we should stop here or 1448 * reclaim more. It's depends on callers. last_scanned_child 1449 * will work enough for keeping fairness under tree. 1450 */ 1451 if (shrink) 1452 return ret; 1453 total += ret; 1454 if (check_soft) { 1455 if (!res_counter_soft_limit_excess(&root_mem->res)) 1456 return total; 1457 } else if (mem_cgroup_margin(root_mem)) 1458 return 1 + total; 1459 } 1460 return total; 1461} 1462 1463/* 1464 * Check OOM-Killer is already running under our hierarchy. 1465 * If someone is running, return false. 1466 */ 1467static bool mem_cgroup_oom_lock(struct mem_cgroup *mem) 1468{ 1469 int x, lock_count = 0; 1470 struct mem_cgroup *iter; 1471 1472 for_each_mem_cgroup_tree(iter, mem) { 1473 x = atomic_inc_return(&iter->oom_lock); 1474 lock_count = max(x, lock_count); 1475 } 1476 1477 if (lock_count == 1) 1478 return true; 1479 return false; 1480} 1481 1482static int mem_cgroup_oom_unlock(struct mem_cgroup *mem) 1483{ 1484 struct mem_cgroup *iter; 1485 1486 /* 1487 * When a new child is created while the hierarchy is under oom, 1488 * mem_cgroup_oom_lock() may not be called. We have to use 1489 * atomic_add_unless() here. 1490 */ 1491 for_each_mem_cgroup_tree(iter, mem) 1492 atomic_add_unless(&iter->oom_lock, -1, 0); 1493 return 0; 1494} 1495 1496 1497static DEFINE_MUTEX(memcg_oom_mutex); 1498static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq); 1499 1500struct oom_wait_info { 1501 struct mem_cgroup *mem; 1502 wait_queue_t wait; 1503}; 1504 1505static int memcg_oom_wake_function(wait_queue_t *wait, 1506 unsigned mode, int sync, void *arg) 1507{ 1508 struct mem_cgroup *wake_mem = (struct mem_cgroup *)arg; 1509 struct oom_wait_info *oom_wait_info; 1510 1511 oom_wait_info = container_of(wait, struct oom_wait_info, wait); 1512 1513 if (oom_wait_info->mem == wake_mem) 1514 goto wakeup; 1515 /* if no hierarchy, no match */ 1516 if (!oom_wait_info->mem->use_hierarchy || !wake_mem->use_hierarchy) 1517 return 0; 1518 /* 1519 * Both of oom_wait_info->mem and wake_mem are stable under us. 1520 * Then we can use css_is_ancestor without taking care of RCU. 1521 */ 1522 if (!css_is_ancestor(&oom_wait_info->mem->css, &wake_mem->css) && 1523 !css_is_ancestor(&wake_mem->css, &oom_wait_info->mem->css)) 1524 return 0; 1525 1526wakeup: 1527 return autoremove_wake_function(wait, mode, sync, arg); 1528} 1529 1530static void memcg_wakeup_oom(struct mem_cgroup *mem) 1531{ 1532 /* for filtering, pass "mem" as argument. */ 1533 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, mem); 1534} 1535 1536static void memcg_oom_recover(struct mem_cgroup *mem) 1537{ 1538 if (mem && atomic_read(&mem->oom_lock)) 1539 memcg_wakeup_oom(mem); 1540} 1541 1542/* 1543 * try to call OOM killer. returns false if we should exit memory-reclaim loop. 1544 */ 1545bool mem_cgroup_handle_oom(struct mem_cgroup *mem, gfp_t mask) 1546{ 1547 struct oom_wait_info owait; 1548 bool locked, need_to_kill; 1549 1550 owait.mem = mem; 1551 owait.wait.flags = 0; 1552 owait.wait.func = memcg_oom_wake_function; 1553 owait.wait.private = current; 1554 INIT_LIST_HEAD(&owait.wait.task_list); 1555 need_to_kill = true; 1556 /* At first, try to OOM lock hierarchy under mem.*/ 1557 mutex_lock(&memcg_oom_mutex); 1558 locked = mem_cgroup_oom_lock(mem); 1559 /* 1560 * Even if signal_pending(), we can't quit charge() loop without 1561 * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL 1562 * under OOM is always welcomed, use TASK_KILLABLE here. 1563 */ 1564 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE); 1565 if (!locked || mem->oom_kill_disable) 1566 need_to_kill = false; 1567 if (locked) 1568 mem_cgroup_oom_notify(mem); 1569 mutex_unlock(&memcg_oom_mutex); 1570 1571 if (need_to_kill) { 1572 finish_wait(&memcg_oom_waitq, &owait.wait); 1573 mem_cgroup_out_of_memory(mem, mask); 1574 } else { 1575 schedule(); 1576 finish_wait(&memcg_oom_waitq, &owait.wait); 1577 } 1578 mutex_lock(&memcg_oom_mutex); 1579 mem_cgroup_oom_unlock(mem); 1580 memcg_wakeup_oom(mem); 1581 mutex_unlock(&memcg_oom_mutex); 1582 1583 if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current)) 1584 return false; 1585 /* Give chance to dying process */ 1586 schedule_timeout(1); 1587 return true; 1588} 1589 1590/* 1591 * Currently used to update mapped file statistics, but the routine can be 1592 * generalized to update other statistics as well. 1593 * 1594 * Notes: Race condition 1595 * 1596 * We usually use page_cgroup_lock() for accessing page_cgroup member but 1597 * it tends to be costly. But considering some conditions, we doesn't need 1598 * to do so _always_. 1599 * 1600 * Considering "charge", lock_page_cgroup() is not required because all 1601 * file-stat operations happen after a page is attached to radix-tree. There 1602 * are no race with "charge". 1603 * 1604 * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup 1605 * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even 1606 * if there are race with "uncharge". Statistics itself is properly handled 1607 * by flags. 1608 * 1609 * Considering "move", this is an only case we see a race. To make the race 1610 * small, we check MEM_CGROUP_ON_MOVE percpu value and detect there are 1611 * possibility of race condition. If there is, we take a lock. 1612 */ 1613 1614void mem_cgroup_update_page_stat(struct page *page, 1615 enum mem_cgroup_page_stat_item idx, int val) 1616{ 1617 struct mem_cgroup *mem; 1618 struct page_cgroup *pc = lookup_page_cgroup(page); 1619 bool need_unlock = false; 1620 unsigned long uninitialized_var(flags); 1621 1622 if (unlikely(!pc)) 1623 return; 1624 1625 rcu_read_lock(); 1626 mem = pc->mem_cgroup; 1627 if (unlikely(!mem || !PageCgroupUsed(pc))) 1628 goto out; 1629 /* pc->mem_cgroup is unstable ? */ 1630 if (unlikely(mem_cgroup_stealed(mem)) || PageTransHuge(page)) { 1631 /* take a lock against to access pc->mem_cgroup */ 1632 move_lock_page_cgroup(pc, &flags); 1633 need_unlock = true; 1634 mem = pc->mem_cgroup; 1635 if (!mem || !PageCgroupUsed(pc)) 1636 goto out; 1637 } 1638 1639 switch (idx) { 1640 case MEMCG_NR_FILE_MAPPED: 1641 if (val > 0) 1642 SetPageCgroupFileMapped(pc); 1643 else if (!page_mapped(page)) 1644 ClearPageCgroupFileMapped(pc); 1645 idx = MEM_CGROUP_STAT_FILE_MAPPED; 1646 break; 1647 default: 1648 BUG(); 1649 } 1650 1651 this_cpu_add(mem->stat->count[idx], val); 1652 1653out: 1654 if (unlikely(need_unlock)) 1655 move_unlock_page_cgroup(pc, &flags); 1656 rcu_read_unlock(); 1657 return; 1658} 1659EXPORT_SYMBOL(mem_cgroup_update_page_stat); 1660 1661/* 1662 * size of first charge trial. "32" comes from vmscan.c's magic value. 1663 * TODO: maybe necessary to use big numbers in big irons. 1664 */ 1665#define CHARGE_SIZE (32 * PAGE_SIZE) 1666struct memcg_stock_pcp { 1667 struct mem_cgroup *cached; /* this never be root cgroup */ 1668 int charge; 1669 struct work_struct work; 1670}; 1671static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock); 1672static atomic_t memcg_drain_count; 1673 1674/* 1675 * Try to consume stocked charge on this cpu. If success, PAGE_SIZE is consumed 1676 * from local stock and true is returned. If the stock is 0 or charges from a 1677 * cgroup which is not current target, returns false. This stock will be 1678 * refilled. 1679 */ 1680static bool consume_stock(struct mem_cgroup *mem) 1681{ 1682 struct memcg_stock_pcp *stock; 1683 bool ret = true; 1684 1685 stock = &get_cpu_var(memcg_stock); 1686 if (mem == stock->cached && stock->charge) 1687 stock->charge -= PAGE_SIZE; 1688 else /* need to call res_counter_charge */ 1689 ret = false; 1690 put_cpu_var(memcg_stock); 1691 return ret; 1692} 1693 1694/* 1695 * Returns stocks cached in percpu to res_counter and reset cached information. 1696 */ 1697static void drain_stock(struct memcg_stock_pcp *stock) 1698{ 1699 struct mem_cgroup *old = stock->cached; 1700 1701 if (stock->charge) { 1702 res_counter_uncharge(&old->res, stock->charge); 1703 if (do_swap_account) 1704 res_counter_uncharge(&old->memsw, stock->charge); 1705 } 1706 stock->cached = NULL; 1707 stock->charge = 0; 1708} 1709 1710/* 1711 * This must be called under preempt disabled or must be called by 1712 * a thread which is pinned to local cpu. 1713 */ 1714static void drain_local_stock(struct work_struct *dummy) 1715{ 1716 struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock); 1717 drain_stock(stock); 1718} 1719 1720/* 1721 * Cache charges(val) which is from res_counter, to local per_cpu area. 1722 * This will be consumed by consume_stock() function, later. 1723 */ 1724static void refill_stock(struct mem_cgroup *mem, int val) 1725{ 1726 struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock); 1727 1728 if (stock->cached != mem) { /* reset if necessary */ 1729 drain_stock(stock); 1730 stock->cached = mem; 1731 } 1732 stock->charge += val; 1733 put_cpu_var(memcg_stock); 1734} 1735 1736/* 1737 * Tries to drain stocked charges in other cpus. This function is asynchronous 1738 * and just put a work per cpu for draining localy on each cpu. Caller can 1739 * expects some charges will be back to res_counter later but cannot wait for 1740 * it. 1741 */ 1742static void drain_all_stock_async(void) 1743{ 1744 int cpu; 1745 /* This function is for scheduling "drain" in asynchronous way. 1746 * The result of "drain" is not directly handled by callers. Then, 1747 * if someone is calling drain, we don't have to call drain more. 1748 * Anyway, WORK_STRUCT_PENDING check in queue_work_on() will catch if 1749 * there is a race. We just do loose check here. 1750 */ 1751 if (atomic_read(&memcg_drain_count)) 1752 return; 1753 /* Notify other cpus that system-wide "drain" is running */ 1754 atomic_inc(&memcg_drain_count); 1755 get_online_cpus(); 1756 for_each_online_cpu(cpu) { 1757 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); 1758 schedule_work_on(cpu, &stock->work); 1759 } 1760 put_online_cpus(); 1761 atomic_dec(&memcg_drain_count); 1762 /* We don't wait for flush_work */ 1763} 1764 1765/* This is a synchronous drain interface. */ 1766static void drain_all_stock_sync(void) 1767{ 1768 /* called when force_empty is called */ 1769 atomic_inc(&memcg_drain_count); 1770 schedule_on_each_cpu(drain_local_stock); 1771 atomic_dec(&memcg_drain_count); 1772} 1773 1774/* 1775 * This function drains percpu counter value from DEAD cpu and 1776 * move it to local cpu. Note that this function can be preempted. 1777 */ 1778static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *mem, int cpu) 1779{ 1780 int i; 1781 1782 spin_lock(&mem->pcp_counter_lock); 1783 for (i = 0; i < MEM_CGROUP_STAT_DATA; i++) { 1784 s64 x = per_cpu(mem->stat->count[i], cpu); 1785 1786 per_cpu(mem->stat->count[i], cpu) = 0; 1787 mem->nocpu_base.count[i] += x; 1788 } 1789 /* need to clear ON_MOVE value, works as a kind of lock. */ 1790 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) = 0; 1791 spin_unlock(&mem->pcp_counter_lock); 1792} 1793 1794static void synchronize_mem_cgroup_on_move(struct mem_cgroup *mem, int cpu) 1795{ 1796 int idx = MEM_CGROUP_ON_MOVE; 1797 1798 spin_lock(&mem->pcp_counter_lock); 1799 per_cpu(mem->stat->count[idx], cpu) = mem->nocpu_base.count[idx]; 1800 spin_unlock(&mem->pcp_counter_lock); 1801} 1802 1803static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb, 1804 unsigned long action, 1805 void *hcpu) 1806{ 1807 int cpu = (unsigned long)hcpu; 1808 struct memcg_stock_pcp *stock; 1809 struct mem_cgroup *iter; 1810 1811 if ((action == CPU_ONLINE)) { 1812 for_each_mem_cgroup_all(iter) 1813 synchronize_mem_cgroup_on_move(iter, cpu); 1814 return NOTIFY_OK; 1815 } 1816 1817 if ((action != CPU_DEAD) || action != CPU_DEAD_FROZEN) 1818 return NOTIFY_OK; 1819 1820 for_each_mem_cgroup_all(iter) 1821 mem_cgroup_drain_pcp_counter(iter, cpu); 1822 1823 stock = &per_cpu(memcg_stock, cpu); 1824 drain_stock(stock); 1825 return NOTIFY_OK; 1826} 1827 1828 1829/* See __mem_cgroup_try_charge() for details */ 1830enum { 1831 CHARGE_OK, /* success */ 1832 CHARGE_RETRY, /* need to retry but retry is not bad */ 1833 CHARGE_NOMEM, /* we can't do more. return -ENOMEM */ 1834 CHARGE_WOULDBLOCK, /* GFP_WAIT wasn't set and no enough res. */ 1835 CHARGE_OOM_DIE, /* the current is killed because of OOM */ 1836}; 1837 1838static int __mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask, 1839 int csize, bool oom_check) 1840{ 1841 struct mem_cgroup *mem_over_limit; 1842 struct res_counter *fail_res; 1843 unsigned long flags = 0; 1844 int ret; 1845 1846 ret = res_counter_charge(&mem->res, csize, &fail_res); 1847 1848 if (likely(!ret)) { 1849 if (!do_swap_account) 1850 return CHARGE_OK; 1851 ret = res_counter_charge(&mem->memsw, csize, &fail_res); 1852 if (likely(!ret)) 1853 return CHARGE_OK; 1854 1855 res_counter_uncharge(&mem->res, csize); 1856 mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw); 1857 flags |= MEM_CGROUP_RECLAIM_NOSWAP; 1858 } else 1859 mem_over_limit = mem_cgroup_from_res_counter(fail_res, res); 1860 /* 1861 * csize can be either a huge page (HPAGE_SIZE), a batch of 1862 * regular pages (CHARGE_SIZE), or a single regular page 1863 * (PAGE_SIZE). 1864 * 1865 * Never reclaim on behalf of optional batching, retry with a 1866 * single page instead. 1867 */ 1868 if (csize == CHARGE_SIZE) 1869 return CHARGE_RETRY; 1870 1871 if (!(gfp_mask & __GFP_WAIT)) 1872 return CHARGE_WOULDBLOCK; 1873 1874 ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL, 1875 gfp_mask, flags); 1876 if (mem_cgroup_margin(mem_over_limit) >= csize) 1877 return CHARGE_RETRY; 1878 /* 1879 * Even though the limit is exceeded at this point, reclaim 1880 * may have been able to free some pages. Retry the charge 1881 * before killing the task. 1882 * 1883 * Only for regular pages, though: huge pages are rather 1884 * unlikely to succeed so close to the limit, and we fall back 1885 * to regular pages anyway in case of failure. 1886 */ 1887 if (csize == PAGE_SIZE && ret) 1888 return CHARGE_RETRY; 1889 1890 /* 1891 * At task move, charge accounts can be doubly counted. So, it's 1892 * better to wait until the end of task_move if something is going on. 1893 */ 1894 if (mem_cgroup_wait_acct_move(mem_over_limit)) 1895 return CHARGE_RETRY; 1896 1897 /* If we don't need to call oom-killer at el, return immediately */ 1898 if (!oom_check) 1899 return CHARGE_NOMEM; 1900 /* check OOM */ 1901 if (!mem_cgroup_handle_oom(mem_over_limit, gfp_mask)) 1902 return CHARGE_OOM_DIE; 1903 1904 return CHARGE_RETRY; 1905} 1906 1907/* 1908 * Unlike exported interface, "oom" parameter is added. if oom==true, 1909 * oom-killer can be invoked. 1910 */ 1911static int __mem_cgroup_try_charge(struct mm_struct *mm, 1912 gfp_t gfp_mask, 1913 struct mem_cgroup **memcg, bool oom, 1914 int page_size) 1915{ 1916 int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES; 1917 struct mem_cgroup *mem = NULL; 1918 int ret; 1919 int csize = max(CHARGE_SIZE, (unsigned long) page_size); 1920 1921 /* 1922 * Unlike gloval-vm's OOM-kill, we're not in memory shortage 1923 * in system level. So, allow to go ahead dying process in addition to 1924 * MEMDIE process. 1925 */ 1926 if (unlikely(test_thread_flag(TIF_MEMDIE) 1927 || fatal_signal_pending(current))) 1928 goto bypass; 1929 1930 /* 1931 * We always charge the cgroup the mm_struct belongs to. 1932 * The mm_struct's mem_cgroup changes on task migration if the 1933 * thread group leader migrates. It's possible that mm is not 1934 * set, if so charge the init_mm (happens for pagecache usage). 1935 */ 1936 if (!*memcg && !mm) 1937 goto bypass; 1938again: 1939 if (*memcg) { /* css should be a valid one */ 1940 mem = *memcg; 1941 VM_BUG_ON(css_is_removed(&mem->css)); 1942 if (mem_cgroup_is_root(mem)) 1943 goto done; 1944 if (page_size == PAGE_SIZE && consume_stock(mem)) 1945 goto done; 1946 css_get(&mem->css); 1947 } else { 1948 struct task_struct *p; 1949 1950 rcu_read_lock(); 1951 p = rcu_dereference(mm->owner); 1952 /* 1953 * Because we don't have task_lock(), "p" can exit. 1954 * In that case, "mem" can point to root or p can be NULL with 1955 * race with swapoff. Then, we have small risk of mis-accouning. 1956 * But such kind of mis-account by race always happens because 1957 * we don't have cgroup_mutex(). It's overkill and we allo that 1958 * small race, here. 1959 * (*) swapoff at el will charge against mm-struct not against 1960 * task-struct. So, mm->owner can be NULL. 1961 */ 1962 mem = mem_cgroup_from_task(p); 1963 if (!mem || mem_cgroup_is_root(mem)) { 1964 rcu_read_unlock(); 1965 goto done; 1966 } 1967 if (page_size == PAGE_SIZE && consume_stock(mem)) { 1968 /* 1969 * It seems dagerous to access memcg without css_get(). 1970 * But considering how consume_stok works, it's not 1971 * necessary. If consume_stock success, some charges 1972 * from this memcg are cached on this cpu. So, we 1973 * don't need to call css_get()/css_tryget() before 1974 * calling consume_stock(). 1975 */ 1976 rcu_read_unlock(); 1977 goto done; 1978 } 1979 /* after here, we may be blocked. we need to get refcnt */ 1980 if (!css_tryget(&mem->css)) { 1981 rcu_read_unlock(); 1982 goto again; 1983 } 1984 rcu_read_unlock(); 1985 } 1986 1987 do { 1988 bool oom_check; 1989 1990 /* If killed, bypass charge */ 1991 if (fatal_signal_pending(current)) { 1992 css_put(&mem->css); 1993 goto bypass; 1994 } 1995 1996 oom_check = false; 1997 if (oom && !nr_oom_retries) { 1998 oom_check = true; 1999 nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES; 2000 } 2001 2002 ret = __mem_cgroup_do_charge(mem, gfp_mask, csize, oom_check); 2003 2004 switch (ret) { 2005 case CHARGE_OK: 2006 break; 2007 case CHARGE_RETRY: /* not in OOM situation but retry */ 2008 csize = page_size; 2009 css_put(&mem->css); 2010 mem = NULL; 2011 goto again; 2012 case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */ 2013 css_put(&mem->css); 2014 goto nomem; 2015 case CHARGE_NOMEM: /* OOM routine works */ 2016 if (!oom) { 2017 css_put(&mem->css); 2018 goto nomem; 2019 } 2020 /* If oom, we never return -ENOMEM */ 2021 nr_oom_retries--; 2022 break; 2023 case CHARGE_OOM_DIE: /* Killed by OOM Killer */ 2024 css_put(&mem->css); 2025 goto bypass; 2026 } 2027 } while (ret != CHARGE_OK); 2028 2029 if (csize > page_size) 2030 refill_stock(mem, csize - page_size); 2031 css_put(&mem->css); 2032done: 2033 *memcg = mem; 2034 return 0; 2035nomem: 2036 *memcg = NULL; 2037 return -ENOMEM; 2038bypass: 2039 *memcg = NULL; 2040 return 0; 2041} 2042 2043/* 2044 * Somemtimes we have to undo a charge we got by try_charge(). 2045 * This function is for that and do uncharge, put css's refcnt. 2046 * gotten by try_charge(). 2047 */ 2048static void __mem_cgroup_cancel_charge(struct mem_cgroup *mem, 2049 unsigned long count) 2050{ 2051 if (!mem_cgroup_is_root(mem)) { 2052 res_counter_uncharge(&mem->res, PAGE_SIZE * count); 2053 if (do_swap_account) 2054 res_counter_uncharge(&mem->memsw, PAGE_SIZE * count); 2055 } 2056} 2057 2058static void mem_cgroup_cancel_charge(struct mem_cgroup *mem, 2059 int page_size) 2060{ 2061 __mem_cgroup_cancel_charge(mem, page_size >> PAGE_SHIFT); 2062} 2063 2064/* 2065 * A helper function to get mem_cgroup from ID. must be called under 2066 * rcu_read_lock(). The caller must check css_is_removed() or some if 2067 * it's concern. (dropping refcnt from swap can be called against removed 2068 * memcg.) 2069 */ 2070static struct mem_cgroup *mem_cgroup_lookup(unsigned short id) 2071{ 2072 struct cgroup_subsys_state *css; 2073 2074 /* ID 0 is unused ID */ 2075 if (!id) 2076 return NULL; 2077 css = css_lookup(&mem_cgroup_subsys, id); 2078 if (!css) 2079 return NULL; 2080 return container_of(css, struct mem_cgroup, css); 2081} 2082 2083struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page) 2084{ 2085 struct mem_cgroup *mem = NULL; 2086 struct page_cgroup *pc; 2087 unsigned short id; 2088 swp_entry_t ent; 2089 2090 VM_BUG_ON(!PageLocked(page)); 2091 2092 pc = lookup_page_cgroup(page); 2093 lock_page_cgroup(pc); 2094 if (PageCgroupUsed(pc)) { 2095 mem = pc->mem_cgroup; 2096 if (mem && !css_tryget(&mem->css)) 2097 mem = NULL; 2098 } else if (PageSwapCache(page)) { 2099 ent.val = page_private(page); 2100 id = lookup_swap_cgroup(ent); 2101 rcu_read_lock(); 2102 mem = mem_cgroup_lookup(id); 2103 if (mem && !css_tryget(&mem->css)) 2104 mem = NULL; 2105 rcu_read_unlock(); 2106 } 2107 unlock_page_cgroup(pc); 2108 return mem; 2109} 2110 2111static void __mem_cgroup_commit_charge(struct mem_cgroup *mem, 2112 struct page_cgroup *pc, 2113 enum charge_type ctype, 2114 int page_size) 2115{ 2116 int nr_pages = page_size >> PAGE_SHIFT; 2117 2118 lock_page_cgroup(pc); 2119 if (unlikely(PageCgroupUsed(pc))) { 2120 unlock_page_cgroup(pc); 2121 mem_cgroup_cancel_charge(mem, page_size); 2122 return; 2123 } 2124 /* 2125 * we don't need page_cgroup_lock about tail pages, becase they are not 2126 * accessed by any other context at this point. 2127 */ 2128 pc->mem_cgroup = mem; 2129 /* 2130 * We access a page_cgroup asynchronously without lock_page_cgroup(). 2131 * Especially when a page_cgroup is taken from a page, pc->mem_cgroup 2132 * is accessed after testing USED bit. To make pc->mem_cgroup visible 2133 * before USED bit, we need memory barrier here. 2134 * See mem_cgroup_add_lru_list(), etc. 2135 */ 2136 smp_wmb(); 2137 switch (ctype) { 2138 case MEM_CGROUP_CHARGE_TYPE_CACHE: 2139 case MEM_CGROUP_CHARGE_TYPE_SHMEM: 2140 SetPageCgroupCache(pc); 2141 SetPageCgroupUsed(pc); 2142 break; 2143 case MEM_CGROUP_CHARGE_TYPE_MAPPED: 2144 ClearPageCgroupCache(pc); 2145 SetPageCgroupUsed(pc); 2146 break; 2147 default: 2148 break; 2149 } 2150 2151 mem_cgroup_charge_statistics(mem, PageCgroupCache(pc), nr_pages); 2152 unlock_page_cgroup(pc); 2153 /* 2154 * "charge_statistics" updated event counter. Then, check it. 2155 * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree. 2156 * if they exceeds softlimit. 2157 */ 2158 memcg_check_events(mem, pc->page); 2159} 2160 2161#ifdef CONFIG_TRANSPARENT_HUGEPAGE 2162 2163#define PCGF_NOCOPY_AT_SPLIT ((1 << PCG_LOCK) | (1 << PCG_MOVE_LOCK) |\ 2164 (1 << PCG_ACCT_LRU) | (1 << PCG_MIGRATION)) 2165/* 2166 * Because tail pages are not marked as "used", set it. We're under 2167 * zone->lru_lock, 'splitting on pmd' and compund_lock. 2168 */ 2169void mem_cgroup_split_huge_fixup(struct page *head, struct page *tail) 2170{ 2171 struct page_cgroup *head_pc = lookup_page_cgroup(head); 2172 struct page_cgroup *tail_pc = lookup_page_cgroup(tail); 2173 unsigned long flags; 2174 2175 if (mem_cgroup_disabled()) 2176 return; 2177 /* 2178 * We have no races with charge/uncharge but will have races with 2179 * page state accounting. 2180 */ 2181 move_lock_page_cgroup(head_pc, &flags); 2182 2183 tail_pc->mem_cgroup = head_pc->mem_cgroup; 2184 smp_wmb(); /* see __commit_charge() */ 2185 if (PageCgroupAcctLRU(head_pc)) { 2186 enum lru_list lru; 2187 struct mem_cgroup_per_zone *mz; 2188 2189 /* 2190 * LRU flags cannot be copied because we need to add tail 2191 *.page to LRU by generic call and our hook will be called. 2192 * We hold lru_lock, then, reduce counter directly. 2193 */ 2194 lru = page_lru(head); 2195 mz = page_cgroup_zoneinfo(head_pc); 2196 MEM_CGROUP_ZSTAT(mz, lru) -= 1; 2197 } 2198 tail_pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT; 2199 move_unlock_page_cgroup(head_pc, &flags); 2200} 2201#endif 2202 2203/** 2204 * __mem_cgroup_move_account - move account of the page 2205 * @pc: page_cgroup of the page. 2206 * @from: mem_cgroup which the page is moved from. 2207 * @to: mem_cgroup which the page is moved to. @from != @to. 2208 * @uncharge: whether we should call uncharge and css_put against @from. 2209 * 2210 * The caller must confirm following. 2211 * - page is not on LRU (isolate_page() is useful.) 2212 * - the pc is locked, used, and ->mem_cgroup points to @from. 2213 * 2214 * This function doesn't do "charge" nor css_get to new cgroup. It should be 2215 * done by a caller(__mem_cgroup_try_charge would be usefull). If @uncharge is 2216 * true, this function does "uncharge" from old cgroup, but it doesn't if 2217 * @uncharge is false, so a caller should do "uncharge". 2218 */ 2219 2220static void __mem_cgroup_move_account(struct page_cgroup *pc, 2221 struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge, 2222 int charge_size) 2223{ 2224 int nr_pages = charge_size >> PAGE_SHIFT; 2225 2226 VM_BUG_ON(from == to); 2227 VM_BUG_ON(PageLRU(pc->page)); 2228 VM_BUG_ON(!page_is_cgroup_locked(pc)); 2229 VM_BUG_ON(!PageCgroupUsed(pc)); 2230 VM_BUG_ON(pc->mem_cgroup != from); 2231 2232 if (PageCgroupFileMapped(pc)) { 2233 /* Update mapped_file data for mem_cgroup */ 2234 preempt_disable(); 2235 __this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]); 2236 __this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]); 2237 preempt_enable(); 2238 } 2239 mem_cgroup_charge_statistics(from, PageCgroupCache(pc), -nr_pages); 2240 if (uncharge) 2241 /* This is not "cancel", but cancel_charge does all we need. */ 2242 mem_cgroup_cancel_charge(from, charge_size); 2243 2244 /* caller should have done css_get */ 2245 pc->mem_cgroup = to; 2246 mem_cgroup_charge_statistics(to, PageCgroupCache(pc), nr_pages); 2247 /* 2248 * We charges against "to" which may not have any tasks. Then, "to" 2249 * can be under rmdir(). But in current implementation, caller of 2250 * this function is just force_empty() and move charge, so it's 2251 * garanteed that "to" is never removed. So, we don't check rmdir 2252 * status here. 2253 */ 2254} 2255 2256/* 2257 * check whether the @pc is valid for moving account and call 2258 * __mem_cgroup_move_account() 2259 */ 2260static int mem_cgroup_move_account(struct page_cgroup *pc, 2261 struct mem_cgroup *from, struct mem_cgroup *to, 2262 bool uncharge, int charge_size) 2263{ 2264 int ret = -EINVAL; 2265 unsigned long flags; 2266 /* 2267 * The page is isolated from LRU. So, collapse function 2268 * will not handle this page. But page splitting can happen. 2269 * Do this check under compound_page_lock(). The caller should 2270 * hold it. 2271 */ 2272 if ((charge_size > PAGE_SIZE) && !PageTransHuge(pc->page)) 2273 return -EBUSY; 2274 2275 lock_page_cgroup(pc); 2276 if (PageCgroupUsed(pc) && pc->mem_cgroup == from) { 2277 move_lock_page_cgroup(pc, &flags); 2278 __mem_cgroup_move_account(pc, from, to, uncharge, charge_size); 2279 move_unlock_page_cgroup(pc, &flags); 2280 ret = 0; 2281 } 2282 unlock_page_cgroup(pc); 2283 /* 2284 * check events 2285 */ 2286 memcg_check_events(to, pc->page); 2287 memcg_check_events(from, pc->page); 2288 return ret; 2289} 2290 2291/* 2292 * move charges to its parent. 2293 */ 2294 2295static int mem_cgroup_move_parent(struct page_cgroup *pc, 2296 struct mem_cgroup *child, 2297 gfp_t gfp_mask) 2298{ 2299 struct page *page = pc->page; 2300 struct cgroup *cg = child->css.cgroup; 2301 struct cgroup *pcg = cg->parent; 2302 struct mem_cgroup *parent; 2303 int page_size = PAGE_SIZE; 2304 unsigned long flags; 2305 int ret; 2306 2307 /* Is ROOT ? */ 2308 if (!pcg) 2309 return -EINVAL; 2310 2311 ret = -EBUSY; 2312 if (!get_page_unless_zero(page)) 2313 goto out; 2314 if (isolate_lru_page(page)) 2315 goto put; 2316 2317 if (PageTransHuge(page)) 2318 page_size = HPAGE_SIZE; 2319 2320 parent = mem_cgroup_from_cont(pcg); 2321 ret = __mem_cgroup_try_charge(NULL, gfp_mask, 2322 &parent, false, page_size); 2323 if (ret || !parent) 2324 goto put_back; 2325 2326 if (page_size > PAGE_SIZE) 2327 flags = compound_lock_irqsave(page); 2328 2329 ret = mem_cgroup_move_account(pc, child, parent, true, page_size); 2330 if (ret) 2331 mem_cgroup_cancel_charge(parent, page_size); 2332 2333 if (page_size > PAGE_SIZE) 2334 compound_unlock_irqrestore(page, flags); 2335put_back: 2336 putback_lru_page(page); 2337put: 2338 put_page(page); 2339out: 2340 return ret; 2341} 2342 2343/* 2344 * Charge the memory controller for page usage. 2345 * Return 2346 * 0 if the charge was successful 2347 * < 0 if the cgroup is over its limit 2348 */ 2349static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm, 2350 gfp_t gfp_mask, enum charge_type ctype) 2351{ 2352 struct mem_cgroup *mem = NULL; 2353 int page_size = PAGE_SIZE; 2354 struct page_cgroup *pc; 2355 bool oom = true; 2356 int ret; 2357 2358 if (PageTransHuge(page)) { 2359 page_size <<= compound_order(page); 2360 VM_BUG_ON(!PageTransHuge(page)); 2361 /* 2362 * Never OOM-kill a process for a huge page. The 2363 * fault handler will fall back to regular pages. 2364 */ 2365 oom = false; 2366 } 2367 2368 pc = lookup_page_cgroup(page); 2369 BUG_ON(!pc); /* XXX: remove this and move pc lookup into commit */ 2370 2371 ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, oom, page_size); 2372 if (ret || !mem) 2373 return ret; 2374 2375 __mem_cgroup_commit_charge(mem, pc, ctype, page_size); 2376 return 0; 2377} 2378 2379int mem_cgroup_newpage_charge(struct page *page, 2380 struct mm_struct *mm, gfp_t gfp_mask) 2381{ 2382 if (mem_cgroup_disabled()) 2383 return 0; 2384 /* 2385 * If already mapped, we don't have to account. 2386 * If page cache, page->mapping has address_space. 2387 * But page->mapping may have out-of-use anon_vma pointer, 2388 * detecit it by PageAnon() check. newly-mapped-anon's page->mapping 2389 * is NULL. 2390 */ 2391 if (page_mapped(page) || (page->mapping && !PageAnon(page))) 2392 return 0; 2393 if (unlikely(!mm)) 2394 mm = &init_mm; 2395 return mem_cgroup_charge_common(page, mm, gfp_mask, 2396 MEM_CGROUP_CHARGE_TYPE_MAPPED); 2397} 2398 2399static void 2400__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr, 2401 enum charge_type ctype); 2402 2403int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm, 2404 gfp_t gfp_mask) 2405{ 2406 int ret; 2407 2408 if (mem_cgroup_disabled()) 2409 return 0; 2410 if (PageCompound(page)) 2411 return 0; 2412 /* 2413 * Corner case handling. This is called from add_to_page_cache() 2414 * in usual. But some FS (shmem) precharges this page before calling it 2415 * and call add_to_page_cache() with GFP_NOWAIT. 2416 * 2417 * For GFP_NOWAIT case, the page may be pre-charged before calling 2418 * add_to_page_cache(). (See shmem.c) check it here and avoid to call 2419 * charge twice. (It works but has to pay a bit larger cost.) 2420 * And when the page is SwapCache, it should take swap information 2421 * into account. This is under lock_page() now. 2422 */ 2423 if (!(gfp_mask & __GFP_WAIT)) { 2424 struct page_cgroup *pc; 2425 2426 pc = lookup_page_cgroup(page); 2427 if (!pc) 2428 return 0; 2429 lock_page_cgroup(pc); 2430 if (PageCgroupUsed(pc)) { 2431 unlock_page_cgroup(pc); 2432 return 0; 2433 } 2434 unlock_page_cgroup(pc); 2435 } 2436 2437 if (unlikely(!mm)) 2438 mm = &init_mm; 2439 2440 if (page_is_file_cache(page)) 2441 return mem_cgroup_charge_common(page, mm, gfp_mask, 2442 MEM_CGROUP_CHARGE_TYPE_CACHE); 2443 2444 /* shmem */ 2445 if (PageSwapCache(page)) { 2446 struct mem_cgroup *mem; 2447 2448 ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem); 2449 if (!ret) 2450 __mem_cgroup_commit_charge_swapin(page, mem, 2451 MEM_CGROUP_CHARGE_TYPE_SHMEM); 2452 } else 2453 ret = mem_cgroup_charge_common(page, mm, gfp_mask, 2454 MEM_CGROUP_CHARGE_TYPE_SHMEM); 2455 2456 return ret; 2457} 2458 2459/* 2460 * While swap-in, try_charge -> commit or cancel, the page is locked. 2461 * And when try_charge() successfully returns, one refcnt to memcg without 2462 * struct page_cgroup is acquired. This refcnt will be consumed by 2463 * "commit()" or removed by "cancel()" 2464 */ 2465int mem_cgroup_try_charge_swapin(struct mm_struct *mm, 2466 struct page *page, 2467 gfp_t mask, struct mem_cgroup **ptr) 2468{ 2469 struct mem_cgroup *mem; 2470 int ret; 2471 2472 *ptr = NULL; 2473 2474 if (mem_cgroup_disabled()) 2475 return 0; 2476 2477 if (!do_swap_account) 2478 goto charge_cur_mm; 2479 /* 2480 * A racing thread's fault, or swapoff, may have already updated 2481 * the pte, and even removed page from swap cache: in those cases 2482 * do_swap_page()'s pte_same() test will fail; but there's also a 2483 * KSM case which does need to charge the page. 2484 */ 2485 if (!PageSwapCache(page)) 2486 goto charge_cur_mm; 2487 mem = try_get_mem_cgroup_from_page(page); 2488 if (!mem) 2489 goto charge_cur_mm; 2490 *ptr = mem; 2491 ret = __mem_cgroup_try_charge(NULL, mask, ptr, true, PAGE_SIZE); 2492 css_put(&mem->css); 2493 return ret; 2494charge_cur_mm: 2495 if (unlikely(!mm)) 2496 mm = &init_mm; 2497 return __mem_cgroup_try_charge(mm, mask, ptr, true, PAGE_SIZE); 2498} 2499 2500static void 2501__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr, 2502 enum charge_type ctype) 2503{ 2504 struct page_cgroup *pc; 2505 2506 if (mem_cgroup_disabled()) 2507 return; 2508 if (!ptr) 2509 return; 2510 cgroup_exclude_rmdir(&ptr->css); 2511 pc = lookup_page_cgroup(page); 2512 mem_cgroup_lru_del_before_commit_swapcache(page); 2513 __mem_cgroup_commit_charge(ptr, pc, ctype, PAGE_SIZE); 2514 mem_cgroup_lru_add_after_commit_swapcache(page); 2515 /* 2516 * Now swap is on-memory. This means this page may be 2517 * counted both as mem and swap....double count. 2518 * Fix it by uncharging from memsw. Basically, this SwapCache is stable 2519 * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page() 2520 * may call delete_from_swap_cache() before reach here. 2521 */ 2522 if (do_swap_account && PageSwapCache(page)) { 2523 swp_entry_t ent = {.val = page_private(page)}; 2524 unsigned short id; 2525 struct mem_cgroup *memcg; 2526 2527 id = swap_cgroup_record(ent, 0); 2528 rcu_read_lock(); 2529 memcg = mem_cgroup_lookup(id); 2530 if (memcg) { 2531 /* 2532 * This recorded memcg can be obsolete one. So, avoid 2533 * calling css_tryget 2534 */ 2535 if (!mem_cgroup_is_root(memcg)) 2536 res_counter_uncharge(&memcg->memsw, PAGE_SIZE); 2537 mem_cgroup_swap_statistics(memcg, false); 2538 mem_cgroup_put(memcg); 2539 } 2540 rcu_read_unlock(); 2541 } 2542 /* 2543 * At swapin, we may charge account against cgroup which has no tasks. 2544 * So, rmdir()->pre_destroy() can be called while we do this charge. 2545 * In that case, we need to call pre_destroy() again. check it here. 2546 */ 2547 cgroup_release_and_wakeup_rmdir(&ptr->css); 2548} 2549 2550void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr) 2551{ 2552 __mem_cgroup_commit_charge_swapin(page, ptr, 2553 MEM_CGROUP_CHARGE_TYPE_MAPPED); 2554} 2555 2556void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem) 2557{ 2558 if (mem_cgroup_disabled()) 2559 return; 2560 if (!mem) 2561 return; 2562 mem_cgroup_cancel_charge(mem, PAGE_SIZE); 2563} 2564 2565static void 2566__do_uncharge(struct mem_cgroup *mem, const enum charge_type ctype, 2567 int page_size) 2568{ 2569 struct memcg_batch_info *batch = NULL; 2570 bool uncharge_memsw = true; 2571 /* If swapout, usage of swap doesn't decrease */ 2572 if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) 2573 uncharge_memsw = false; 2574 2575 batch = ¤t->memcg_batch; 2576 /* 2577 * In usual, we do css_get() when we remember memcg pointer. 2578 * But in this case, we keep res->usage until end of a series of 2579 * uncharges. Then, it's ok to ignore memcg's refcnt. 2580 */ 2581 if (!batch->memcg) 2582 batch->memcg = mem; 2583 /* 2584 * do_batch > 0 when unmapping pages or inode invalidate/truncate. 2585 * In those cases, all pages freed continously can be expected to be in 2586 * the same cgroup and we have chance to coalesce uncharges. 2587 * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE) 2588 * because we want to do uncharge as soon as possible. 2589 */ 2590 2591 if (!batch->do_batch || test_thread_flag(TIF_MEMDIE)) 2592 goto direct_uncharge; 2593 2594 if (page_size != PAGE_SIZE) 2595 goto direct_uncharge; 2596 2597 /* 2598 * In typical case, batch->memcg == mem. This means we can 2599 * merge a series of uncharges to an uncharge of res_counter. 2600 * If not, we uncharge res_counter ony by one. 2601 */ 2602 if (batch->memcg != mem) 2603 goto direct_uncharge; 2604 /* remember freed charge and uncharge it later */ 2605 batch->bytes += PAGE_SIZE; 2606 if (uncharge_memsw) 2607 batch->memsw_bytes += PAGE_SIZE; 2608 return; 2609direct_uncharge: 2610 res_counter_uncharge(&mem->res, page_size); 2611 if (uncharge_memsw) 2612 res_counter_uncharge(&mem->memsw, page_size); 2613 if (unlikely(batch->memcg != mem)) 2614 memcg_oom_recover(mem); 2615 return; 2616} 2617 2618/* 2619 * uncharge if !page_mapped(page) 2620 */ 2621static struct mem_cgroup * 2622__mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype) 2623{ 2624 int count; 2625 struct page_cgroup *pc; 2626 struct mem_cgroup *mem = NULL; 2627 int page_size = PAGE_SIZE; 2628 2629 if (mem_cgroup_disabled()) 2630 return NULL; 2631 2632 if (PageSwapCache(page)) 2633 return NULL; 2634 2635 if (PageTransHuge(page)) { 2636 page_size <<= compound_order(page); 2637 VM_BUG_ON(!PageTransHuge(page)); 2638 } 2639 2640 count = page_size >> PAGE_SHIFT; 2641 /* 2642 * Check if our page_cgroup is valid 2643 */ 2644 pc = lookup_page_cgroup(page); 2645 if (unlikely(!pc || !PageCgroupUsed(pc))) 2646 return NULL; 2647 2648 lock_page_cgroup(pc); 2649 2650 mem = pc->mem_cgroup; 2651 2652 if (!PageCgroupUsed(pc)) 2653 goto unlock_out; 2654 2655 switch (ctype) { 2656 case MEM_CGROUP_CHARGE_TYPE_MAPPED: 2657 case MEM_CGROUP_CHARGE_TYPE_DROP: 2658 /* See mem_cgroup_prepare_migration() */ 2659 if (page_mapped(page) || PageCgroupMigration(pc)) 2660 goto unlock_out; 2661 break; 2662 case MEM_CGROUP_CHARGE_TYPE_SWAPOUT: 2663 if (!PageAnon(page)) { /* Shared memory */ 2664 if (page->mapping && !page_is_file_cache(page)) 2665 goto unlock_out; 2666 } else if (page_mapped(page)) /* Anon */ 2667 goto unlock_out; 2668 break; 2669 default: 2670 break; 2671 } 2672 2673 mem_cgroup_charge_statistics(mem, PageCgroupCache(pc), -count); 2674 2675 ClearPageCgroupUsed(pc); 2676 /* 2677 * pc->mem_cgroup is not cleared here. It will be accessed when it's 2678 * freed from LRU. This is safe because uncharged page is expected not 2679 * to be reused (freed soon). Exception is SwapCache, it's handled by 2680 * special functions. 2681 */ 2682 2683 unlock_page_cgroup(pc); 2684 /* 2685 * even after unlock, we have mem->res.usage here and this memcg 2686 * will never be freed. 2687 */ 2688 memcg_check_events(mem, page); 2689 if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) { 2690 mem_cgroup_swap_statistics(mem, true); 2691 mem_cgroup_get(mem); 2692 } 2693 if (!mem_cgroup_is_root(mem)) 2694 __do_uncharge(mem, ctype, page_size); 2695 2696 return mem; 2697 2698unlock_out: 2699 unlock_page_cgroup(pc); 2700 return NULL; 2701} 2702 2703void mem_cgroup_uncharge_page(struct page *page) 2704{ 2705 /* early check. */ 2706 if (page_mapped(page)) 2707 return; 2708 if (page->mapping && !PageAnon(page)) 2709 return; 2710 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED); 2711} 2712 2713void mem_cgroup_uncharge_cache_page(struct page *page) 2714{ 2715 VM_BUG_ON(page_mapped(page)); 2716 VM_BUG_ON(page->mapping); 2717 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE); 2718} 2719 2720/* 2721 * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate. 2722 * In that cases, pages are freed continuously and we can expect pages 2723 * are in the same memcg. All these calls itself limits the number of 2724 * pages freed at once, then uncharge_start/end() is called properly. 2725 * This may be called prural(2) times in a context, 2726 */ 2727 2728void mem_cgroup_uncharge_start(void) 2729{ 2730 current->memcg_batch.do_batch++; 2731 /* We can do nest. */ 2732 if (current->memcg_batch.do_batch == 1) { 2733 current->memcg_batch.memcg = NULL; 2734 current->memcg_batch.bytes = 0; 2735 current->memcg_batch.memsw_bytes = 0; 2736 } 2737} 2738 2739void mem_cgroup_uncharge_end(void) 2740{ 2741 struct memcg_batch_info *batch = ¤t->memcg_batch; 2742 2743 if (!batch->do_batch) 2744 return; 2745 2746 batch->do_batch--; 2747 if (batch->do_batch) /* If stacked, do nothing. */ 2748 return; 2749 2750 if (!batch->memcg) 2751 return; 2752 /* 2753 * This "batch->memcg" is valid without any css_get/put etc... 2754 * bacause we hide charges behind us. 2755 */ 2756 if (batch->bytes) 2757 res_counter_uncharge(&batch->memcg->res, batch->bytes); 2758 if (batch->memsw_bytes) 2759 res_counter_uncharge(&batch->memcg->memsw, batch->memsw_bytes); 2760 memcg_oom_recover(batch->memcg); 2761 /* forget this pointer (for sanity check) */ 2762 batch->memcg = NULL; 2763} 2764 2765#ifdef CONFIG_SWAP 2766/* 2767 * called after __delete_from_swap_cache() and drop "page" account. 2768 * memcg information is recorded to swap_cgroup of "ent" 2769 */ 2770void 2771mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout) 2772{ 2773 struct mem_cgroup *memcg; 2774 int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT; 2775 2776 if (!swapout) /* this was a swap cache but the swap is unused ! */ 2777 ctype = MEM_CGROUP_CHARGE_TYPE_DROP; 2778 2779 memcg = __mem_cgroup_uncharge_common(page, ctype); 2780 2781 /* 2782 * record memcg information, if swapout && memcg != NULL, 2783 * mem_cgroup_get() was called in uncharge(). 2784 */ 2785 if (do_swap_account && swapout && memcg) 2786 swap_cgroup_record(ent, css_id(&memcg->css)); 2787} 2788#endif 2789 2790#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP 2791/* 2792 * called from swap_entry_free(). remove record in swap_cgroup and 2793 * uncharge "memsw" account. 2794 */ 2795void mem_cgroup_uncharge_swap(swp_entry_t ent) 2796{ 2797 struct mem_cgroup *memcg; 2798 unsigned short id; 2799 2800 if (!do_swap_account) 2801 return; 2802 2803 id = swap_cgroup_record(ent, 0); 2804 rcu_read_lock(); 2805 memcg = mem_cgroup_lookup(id); 2806 if (memcg) { 2807 /* 2808 * We uncharge this because swap is freed. 2809 * This memcg can be obsolete one. We avoid calling css_tryget 2810 */ 2811 if (!mem_cgroup_is_root(memcg)) 2812 res_counter_uncharge(&memcg->memsw, PAGE_SIZE); 2813 mem_cgroup_swap_statistics(memcg, false); 2814 mem_cgroup_put(memcg); 2815 } 2816 rcu_read_unlock(); 2817} 2818 2819/** 2820 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record. 2821 * @entry: swap entry to be moved 2822 * @from: mem_cgroup which the entry is moved from 2823 * @to: mem_cgroup which the entry is moved to 2824 * @need_fixup: whether we should fixup res_counters and refcounts. 2825 * 2826 * It succeeds only when the swap_cgroup's record for this entry is the same 2827 * as the mem_cgroup's id of @from. 2828 * 2829 * Returns 0 on success, -EINVAL on failure. 2830 * 2831 * The caller must have charged to @to, IOW, called res_counter_charge() about 2832 * both res and memsw, and called css_get(). 2833 */ 2834static int mem_cgroup_move_swap_account(swp_entry_t entry, 2835 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup) 2836{ 2837 unsigned short old_id, new_id; 2838 2839 old_id = css_id(&from->css); 2840 new_id = css_id(&to->css); 2841 2842 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) { 2843 mem_cgroup_swap_statistics(from, false); 2844 mem_cgroup_swap_statistics(to, true); 2845 /* 2846 * This function is only called from task migration context now. 2847 * It postpones res_counter and refcount handling till the end 2848 * of task migration(mem_cgroup_clear_mc()) for performance 2849 * improvement. But we cannot postpone mem_cgroup_get(to) 2850 * because if the process that has been moved to @to does 2851 * swap-in, the refcount of @to might be decreased to 0. 2852 */ 2853 mem_cgroup_get(to); 2854 if (need_fixup) { 2855 if (!mem_cgroup_is_root(from)) 2856 res_counter_uncharge(&from->memsw, PAGE_SIZE); 2857 mem_cgroup_put(from); 2858 /* 2859 * we charged both to->res and to->memsw, so we should 2860 * uncharge to->res. 2861 */ 2862 if (!mem_cgroup_is_root(to)) 2863 res_counter_uncharge(&to->res, PAGE_SIZE); 2864 } 2865 return 0; 2866 } 2867 return -EINVAL; 2868} 2869#else 2870static inline int mem_cgroup_move_swap_account(swp_entry_t entry, 2871 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup) 2872{ 2873 return -EINVAL; 2874} 2875#endif 2876 2877/* 2878 * Before starting migration, account PAGE_SIZE to mem_cgroup that the old 2879 * page belongs to. 2880 */ 2881int mem_cgroup_prepare_migration(struct page *page, 2882 struct page *newpage, struct mem_cgroup **ptr, gfp_t gfp_mask) 2883{ 2884 struct page_cgroup *pc; 2885 struct mem_cgroup *mem = NULL; 2886 enum charge_type ctype; 2887 int ret = 0; 2888 2889 *ptr = NULL; 2890 2891 VM_BUG_ON(PageTransHuge(page)); 2892 if (mem_cgroup_disabled()) 2893 return 0; 2894 2895 pc = lookup_page_cgroup(page); 2896 lock_page_cgroup(pc); 2897 if (PageCgroupUsed(pc)) { 2898 mem = pc->mem_cgroup; 2899 css_get(&mem->css); 2900 /* 2901 * At migrating an anonymous page, its mapcount goes down 2902 * to 0 and uncharge() will be called. But, even if it's fully 2903 * unmapped, migration may fail and this page has to be 2904 * charged again. We set MIGRATION flag here and delay uncharge 2905 * until end_migration() is called 2906 * 2907 * Corner Case Thinking 2908 * A) 2909 * When the old page was mapped as Anon and it's unmap-and-freed 2910 * while migration was ongoing. 2911 * If unmap finds the old page, uncharge() of it will be delayed 2912 * until end_migration(). If unmap finds a new page, it's 2913 * uncharged when it make mapcount to be 1->0. If unmap code 2914 * finds swap_migration_entry, the new page will not be mapped 2915 * and end_migration() will find it(mapcount==0). 2916 * 2917 * B) 2918 * When the old page was mapped but migraion fails, the kernel 2919 * remaps it. A charge for it is kept by MIGRATION flag even 2920 * if mapcount goes down to 0. We can do remap successfully 2921 * without charging it again. 2922 * 2923 * C) 2924 * The "old" page is under lock_page() until the end of 2925 * migration, so, the old page itself will not be swapped-out. 2926 * If the new page is swapped out before end_migraton, our 2927 * hook to usual swap-out path will catch the event. 2928 */ 2929 if (PageAnon(page)) 2930 SetPageCgroupMigration(pc); 2931 } 2932 unlock_page_cgroup(pc); 2933 /* 2934 * If the page is not charged at this point, 2935 * we return here. 2936 */ 2937 if (!mem) 2938 return 0; 2939 2940 *ptr = mem; 2941 ret = __mem_cgroup_try_charge(NULL, gfp_mask, ptr, false, PAGE_SIZE); 2942 css_put(&mem->css);/* drop extra refcnt */ 2943 if (ret || *ptr == NULL) { 2944 if (PageAnon(page)) { 2945 lock_page_cgroup(pc); 2946 ClearPageCgroupMigration(pc); 2947 unlock_page_cgroup(pc); 2948 /* 2949 * The old page may be fully unmapped while we kept it. 2950 */ 2951 mem_cgroup_uncharge_page(page); 2952 } 2953 return -ENOMEM; 2954 } 2955 /* 2956 * We charge new page before it's used/mapped. So, even if unlock_page() 2957 * is called before end_migration, we can catch all events on this new 2958 * page. In the case new page is migrated but not remapped, new page's 2959 * mapcount will be finally 0 and we call uncharge in end_migration(). 2960 */ 2961 pc = lookup_page_cgroup(newpage); 2962 if (PageAnon(page)) 2963 ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED; 2964 else if (page_is_file_cache(page)) 2965 ctype = MEM_CGROUP_CHARGE_TYPE_CACHE; 2966 else 2967 ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM; 2968 __mem_cgroup_commit_charge(mem, pc, ctype, PAGE_SIZE); 2969 return ret; 2970} 2971 2972/* remove redundant charge if migration failed*/ 2973void mem_cgroup_end_migration(struct mem_cgroup *mem, 2974 struct page *oldpage, struct page *newpage, bool migration_ok) 2975{ 2976 struct page *used, *unused; 2977 struct page_cgroup *pc; 2978 2979 if (!mem) 2980 return; 2981 /* blocks rmdir() */ 2982 cgroup_exclude_rmdir(&mem->css); 2983 if (!migration_ok) { 2984 used = oldpage; 2985 unused = newpage; 2986 } else { 2987 used = newpage; 2988 unused = oldpage; 2989 } 2990 /* 2991 * We disallowed uncharge of pages under migration because mapcount 2992 * of the page goes down to zero, temporarly. 2993 * Clear the flag and check the page should be charged. 2994 */ 2995 pc = lookup_page_cgroup(oldpage); 2996 lock_page_cgroup(pc); 2997 ClearPageCgroupMigration(pc); 2998 unlock_page_cgroup(pc); 2999 3000 __mem_cgroup_uncharge_common(unused, MEM_CGROUP_CHARGE_TYPE_FORCE); 3001 3002 /* 3003 * If a page is a file cache, radix-tree replacement is very atomic 3004 * and we can skip this check. When it was an Anon page, its mapcount 3005 * goes down to 0. But because we added MIGRATION flage, it's not 3006 * uncharged yet. There are several case but page->mapcount check 3007 * and USED bit check in mem_cgroup_uncharge_page() will do enough 3008 * check. (see prepare_charge() also) 3009 */ 3010 if (PageAnon(used)) 3011 mem_cgroup_uncharge_page(used); 3012 /* 3013 * At migration, we may charge account against cgroup which has no 3014 * tasks. 3015 * So, rmdir()->pre_destroy() can be called while we do this charge. 3016 * In that case, we need to call pre_destroy() again. check it here. 3017 */ 3018 cgroup_release_and_wakeup_rmdir(&mem->css); 3019} 3020 3021/* 3022 * A call to try to shrink memory usage on charge failure at shmem's swapin. 3023 * Calling hierarchical_reclaim is not enough because we should update 3024 * last_oom_jiffies to prevent pagefault_out_of_memory from invoking global OOM. 3025 * Moreover considering hierarchy, we should reclaim from the mem_over_limit, 3026 * not from the memcg which this page would be charged to. 3027 * try_charge_swapin does all of these works properly. 3028 */ 3029int mem_cgroup_shmem_charge_fallback(struct page *page, 3030 struct mm_struct *mm, 3031 gfp_t gfp_mask) 3032{ 3033 struct mem_cgroup *mem; 3034 int ret; 3035 3036 if (mem_cgroup_disabled()) 3037 return 0; 3038 3039 ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem); 3040 if (!ret) 3041 mem_cgroup_cancel_charge_swapin(mem); /* it does !mem check */ 3042 3043 return ret; 3044} 3045 3046#ifdef CONFIG_DEBUG_VM 3047static struct page_cgroup *lookup_page_cgroup_used(struct page *page) 3048{ 3049 struct page_cgroup *pc; 3050 3051 pc = lookup_page_cgroup(page); 3052 if (likely(pc) && PageCgroupUsed(pc)) 3053 return pc; 3054 return NULL; 3055} 3056 3057bool mem_cgroup_bad_page_check(struct page *page) 3058{ 3059 if (mem_cgroup_disabled()) 3060 return false; 3061 3062 return lookup_page_cgroup_used(page) != NULL; 3063} 3064 3065void mem_cgroup_print_bad_page(struct page *page) 3066{ 3067 struct page_cgroup *pc; 3068 3069 pc = lookup_page_cgroup_used(page); 3070 if (pc) { 3071 int ret = -1; 3072 char *path; 3073 3074 printk(KERN_ALERT "pc:%p pc->flags:%lx pc->mem_cgroup:%p", 3075 pc, pc->flags, pc->mem_cgroup); 3076 3077 path = kmalloc(PATH_MAX, GFP_KERNEL); 3078 if (path) { 3079 rcu_read_lock(); 3080 ret = cgroup_path(pc->mem_cgroup->css.cgroup, 3081 path, PATH_MAX); 3082 rcu_read_unlock(); 3083 } 3084 3085 printk(KERN_CONT "(%s)\n", 3086 (ret < 0) ? "cannot get the path" : path); 3087 kfree(path); 3088 } 3089} 3090#endif 3091 3092static DEFINE_MUTEX(set_limit_mutex); 3093 3094static int mem_cgroup_resize_limit(struct mem_cgroup *memcg, 3095 unsigned long long val) 3096{ 3097 int retry_count; 3098 u64 memswlimit, memlimit; 3099 int ret = 0; 3100 int children = mem_cgroup_count_children(memcg); 3101 u64 curusage, oldusage; 3102 int enlarge; 3103 3104 /* 3105 * For keeping hierarchical_reclaim simple, how long we should retry 3106 * is depends on callers. We set our retry-count to be function 3107 * of # of children which we should visit in this loop. 3108 */ 3109 retry_count = MEM_CGROUP_RECLAIM_RETRIES * children; 3110 3111 oldusage = res_counter_read_u64(&memcg->res, RES_USAGE); 3112 3113 enlarge = 0; 3114 while (retry_count) { 3115 if (signal_pending(current)) { 3116 ret = -EINTR; 3117 break; 3118 } 3119 /* 3120 * Rather than hide all in some function, I do this in 3121 * open coded manner. You see what this really does. 3122 * We have to guarantee mem->res.limit < mem->memsw.limit. 3123 */ 3124 mutex_lock(&set_limit_mutex); 3125 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 3126 if (memswlimit < val) { 3127 ret = -EINVAL; 3128 mutex_unlock(&set_limit_mutex); 3129 break; 3130 } 3131 3132 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT); 3133 if (memlimit < val) 3134 enlarge = 1; 3135 3136 ret = res_counter_set_limit(&memcg->res, val); 3137 if (!ret) { 3138 if (memswlimit == val) 3139 memcg->memsw_is_minimum = true; 3140 else 3141 memcg->memsw_is_minimum = false; 3142 } 3143 mutex_unlock(&set_limit_mutex); 3144 3145 if (!ret) 3146 break; 3147 3148 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL, 3149 MEM_CGROUP_RECLAIM_SHRINK); 3150 curusage = res_counter_read_u64(&memcg->res, RES_USAGE); 3151 /* Usage is reduced ? */ 3152 if (curusage >= oldusage) 3153 retry_count--; 3154 else 3155 oldusage = curusage; 3156 } 3157 if (!ret && enlarge) 3158 memcg_oom_recover(memcg); 3159 3160 return ret; 3161} 3162 3163static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg, 3164 unsigned long long val) 3165{ 3166 int retry_count; 3167 u64 memlimit, memswlimit, oldusage, curusage; 3168 int children = mem_cgroup_count_children(memcg); 3169 int ret = -EBUSY; 3170 int enlarge = 0; 3171 3172 /* see mem_cgroup_resize_res_limit */ 3173 retry_count = children * MEM_CGROUP_RECLAIM_RETRIES; 3174 oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE); 3175 while (retry_count) { 3176 if (signal_pending(current)) { 3177 ret = -EINTR; 3178 break; 3179 } 3180 /* 3181 * Rather than hide all in some function, I do this in 3182 * open coded manner. You see what this really does. 3183 * We have to guarantee mem->res.limit < mem->memsw.limit. 3184 */ 3185 mutex_lock(&set_limit_mutex); 3186 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT); 3187 if (memlimit > val) { 3188 ret = -EINVAL; 3189 mutex_unlock(&set_limit_mutex); 3190 break; 3191 } 3192 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 3193 if (memswlimit < val) 3194 enlarge = 1; 3195 ret = res_counter_set_limit(&memcg->memsw, val); 3196 if (!ret) { 3197 if (memlimit == val) 3198 memcg->memsw_is_minimum = true; 3199 else 3200 memcg->memsw_is_minimum = false; 3201 } 3202 mutex_unlock(&set_limit_mutex); 3203 3204 if (!ret) 3205 break; 3206 3207 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL, 3208 MEM_CGROUP_RECLAIM_NOSWAP | 3209 MEM_CGROUP_RECLAIM_SHRINK); 3210 curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE); 3211 /* Usage is reduced ? */ 3212 if (curusage >= oldusage) 3213 retry_count--; 3214 else 3215 oldusage = curusage; 3216 } 3217 if (!ret && enlarge) 3218 memcg_oom_recover(memcg); 3219 return ret; 3220} 3221 3222unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order, 3223 gfp_t gfp_mask) 3224{ 3225 unsigned long nr_reclaimed = 0; 3226 struct mem_cgroup_per_zone *mz, *next_mz = NULL; 3227 unsigned long reclaimed; 3228 int loop = 0; 3229 struct mem_cgroup_tree_per_zone *mctz; 3230 unsigned long long excess; 3231 3232 if (order > 0) 3233 return 0; 3234 3235 mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone)); 3236 /* 3237 * This loop can run a while, specially if mem_cgroup's continuously 3238 * keep exceeding their soft limit and putting the system under 3239 * pressure 3240 */ 3241 do { 3242 if (next_mz) 3243 mz = next_mz; 3244 else 3245 mz = mem_cgroup_largest_soft_limit_node(mctz); 3246 if (!mz) 3247 break; 3248 3249 reclaimed = mem_cgroup_hierarchical_reclaim(mz->mem, zone, 3250 gfp_mask, 3251 MEM_CGROUP_RECLAIM_SOFT); 3252 nr_reclaimed += reclaimed; 3253 spin_lock(&mctz->lock); 3254 3255 /* 3256 * If we failed to reclaim anything from this memory cgroup 3257 * it is time to move on to the next cgroup 3258 */ 3259 next_mz = NULL; 3260 if (!reclaimed) { 3261 do { 3262 /* 3263 * Loop until we find yet another one. 3264 * 3265 * By the time we get the soft_limit lock 3266 * again, someone might have aded the 3267 * group back on the RB tree. Iterate to 3268 * make sure we get a different mem. 3269 * mem_cgroup_largest_soft_limit_node returns 3270 * NULL if no other cgroup is present on 3271 * the tree 3272 */ 3273 next_mz = 3274 __mem_cgroup_largest_soft_limit_node(mctz); 3275 if (next_mz == mz) { 3276 css_put(&next_mz->mem->css); 3277 next_mz = NULL; 3278 } else /* next_mz == NULL or other memcg */ 3279 break; 3280 } while (1); 3281 } 3282 __mem_cgroup_remove_exceeded(mz->mem, mz, mctz); 3283 excess = res_counter_soft_limit_excess(&mz->mem->res); 3284 /* 3285 * One school of thought says that we should not add 3286 * back the node to the tree if reclaim returns 0. 3287 * But our reclaim could return 0, simply because due 3288 * to priority we are exposing a smaller subset of 3289 * memory to reclaim from. Consider this as a longer 3290 * term TODO. 3291 */ 3292 /* If excess == 0, no tree ops */ 3293 __mem_cgroup_insert_exceeded(mz->mem, mz, mctz, excess); 3294 spin_unlock(&mctz->lock); 3295 css_put(&mz->mem->css); 3296 loop++; 3297 /* 3298 * Could not reclaim anything and there are no more 3299 * mem cgroups to try or we seem to be looping without 3300 * reclaiming anything. 3301 */ 3302 if (!nr_reclaimed && 3303 (next_mz == NULL || 3304 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS)) 3305 break; 3306 } while (!nr_reclaimed); 3307 if (next_mz) 3308 css_put(&next_mz->mem->css); 3309 return nr_reclaimed; 3310} 3311 3312/* 3313 * This routine traverse page_cgroup in given list and drop them all. 3314 * *And* this routine doesn't reclaim page itself, just removes page_cgroup. 3315 */ 3316static int mem_cgroup_force_empty_list(struct mem_cgroup *mem, 3317 int node, int zid, enum lru_list lru) 3318{ 3319 struct zone *zone; 3320 struct mem_cgroup_per_zone *mz; 3321 struct page_cgroup *pc, *busy; 3322 unsigned long flags, loop; 3323 struct list_head *list; 3324 int ret = 0; 3325 3326 zone = &NODE_DATA(node)->node_zones[zid]; 3327 mz = mem_cgroup_zoneinfo(mem, node, zid); 3328 list = &mz->lists[lru]; 3329 3330 loop = MEM_CGROUP_ZSTAT(mz, lru); 3331 /* give some margin against EBUSY etc...*/ 3332 loop += 256; 3333 busy = NULL; 3334 while (loop--) { 3335 ret = 0; 3336 spin_lock_irqsave(&zone->lru_lock, flags); 3337 if (list_empty(list)) { 3338 spin_unlock_irqrestore(&zone->lru_lock, flags); 3339 break; 3340 } 3341 pc = list_entry(list->prev, struct page_cgroup, lru); 3342 if (busy == pc) { 3343 list_move(&pc->lru, list); 3344 busy = NULL; 3345 spin_unlock_irqrestore(&zone->lru_lock, flags); 3346 continue; 3347 } 3348 spin_unlock_irqrestore(&zone->lru_lock, flags); 3349 3350 ret = mem_cgroup_move_parent(pc, mem, GFP_KERNEL); 3351 if (ret == -ENOMEM) 3352 break; 3353 3354 if (ret == -EBUSY || ret == -EINVAL) { 3355 /* found lock contention or "pc" is obsolete. */ 3356 busy = pc; 3357 cond_resched(); 3358 } else 3359 busy = NULL; 3360 } 3361 3362 if (!ret && !list_empty(list)) 3363 return -EBUSY; 3364 return ret; 3365} 3366 3367/* 3368 * make mem_cgroup's charge to be 0 if there is no task. 3369 * This enables deleting this mem_cgroup. 3370 */ 3371static int mem_cgroup_force_empty(struct mem_cgroup *mem, bool free_all) 3372{ 3373 int ret; 3374 int node, zid, shrink; 3375 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES; 3376 struct cgroup *cgrp = mem->css.cgroup; 3377 3378 css_get(&mem->css); 3379 3380 shrink = 0; 3381 /* should free all ? */ 3382 if (free_all) 3383 goto try_to_free; 3384move_account: 3385 do { 3386 ret = -EBUSY; 3387 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children)) 3388 goto out; 3389 ret = -EINTR; 3390 if (signal_pending(current)) 3391 goto out; 3392 /* This is for making all *used* pages to be on LRU. */ 3393 lru_add_drain_all(); 3394 drain_all_stock_sync(); 3395 ret = 0; 3396 mem_cgroup_start_move(mem); 3397 for_each_node_state(node, N_HIGH_MEMORY) { 3398 for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) { 3399 enum lru_list l; 3400 for_each_lru(l) { 3401 ret = mem_cgroup_force_empty_list(mem, 3402 node, zid, l); 3403 if (ret) 3404 break; 3405 } 3406 } 3407 if (ret) 3408 break; 3409 } 3410 mem_cgroup_end_move(mem); 3411 memcg_oom_recover(mem); 3412 /* it seems parent cgroup doesn't have enough mem */ 3413 if (ret == -ENOMEM) 3414 goto try_to_free; 3415 cond_resched(); 3416 /* "ret" should also be checked to ensure all lists are empty. */ 3417 } while (mem->res.usage > 0 || ret); 3418out: 3419 css_put(&mem->css); 3420 return ret; 3421 3422try_to_free: 3423 /* returns EBUSY if there is a task or if we come here twice. */ 3424 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children) || shrink) { 3425 ret = -EBUSY; 3426 goto out; 3427 } 3428 /* we call try-to-free pages for make this cgroup empty */ 3429 lru_add_drain_all(); 3430 /* try to free all pages in this cgroup */ 3431 shrink = 1; 3432 while (nr_retries && mem->res.usage > 0) { 3433 int progress; 3434 3435 if (signal_pending(current)) { 3436 ret = -EINTR; 3437 goto out; 3438 } 3439 progress = try_to_free_mem_cgroup_pages(mem, GFP_KERNEL, 3440 false, get_swappiness(mem)); 3441 if (!progress) { 3442 nr_retries--; 3443 /* maybe some writeback is necessary */ 3444 congestion_wait(BLK_RW_ASYNC, HZ/10); 3445 } 3446 3447 } 3448 lru_add_drain(); 3449 /* try move_account...there may be some *locked* pages. */ 3450 goto move_account; 3451} 3452 3453int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event) 3454{ 3455 return mem_cgroup_force_empty(mem_cgroup_from_cont(cont), true); 3456} 3457 3458 3459static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft) 3460{ 3461 return mem_cgroup_from_cont(cont)->use_hierarchy; 3462} 3463 3464static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft, 3465 u64 val) 3466{ 3467 int retval = 0; 3468 struct mem_cgroup *mem = mem_cgroup_from_cont(cont); 3469 struct cgroup *parent = cont->parent; 3470 struct mem_cgroup *parent_mem = NULL; 3471 3472 if (parent) 3473 parent_mem = mem_cgroup_from_cont(parent); 3474 3475 cgroup_lock(); 3476 /* 3477 * If parent's use_hierarchy is set, we can't make any modifications 3478 * in the child subtrees. If it is unset, then the change can 3479 * occur, provided the current cgroup has no children. 3480 * 3481 * For the root cgroup, parent_mem is NULL, we allow value to be 3482 * set if there are no children. 3483 */ 3484 if ((!parent_mem || !parent_mem->use_hierarchy) && 3485 (val == 1 || val == 0)) { 3486 if (list_empty(&cont->children)) 3487 mem->use_hierarchy = val; 3488 else 3489 retval = -EBUSY; 3490 } else 3491 retval = -EINVAL; 3492 cgroup_unlock(); 3493 3494 return retval; 3495} 3496 3497 3498static u64 mem_cgroup_get_recursive_idx_stat(struct mem_cgroup *mem, 3499 enum mem_cgroup_stat_index idx) 3500{ 3501 struct mem_cgroup *iter; 3502 s64 val = 0; 3503 3504 /* each per cpu's value can be minus.Then, use s64 */ 3505 for_each_mem_cgroup_tree(iter, mem) 3506 val += mem_cgroup_read_stat(iter, idx); 3507 3508 if (val < 0) /* race ? */ 3509 val = 0; 3510 return val; 3511} 3512 3513static inline u64 mem_cgroup_usage(struct mem_cgroup *mem, bool swap) 3514{ 3515 u64 val; 3516 3517 if (!mem_cgroup_is_root(mem)) { 3518 if (!swap) 3519 return res_counter_read_u64(&mem->res, RES_USAGE); 3520 else 3521 return res_counter_read_u64(&mem->memsw, RES_USAGE); 3522 } 3523 3524 val = mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_CACHE); 3525 val += mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_RSS); 3526 3527 if (swap) 3528 val += mem_cgroup_get_recursive_idx_stat(mem, 3529 MEM_CGROUP_STAT_SWAPOUT); 3530 3531 return val << PAGE_SHIFT; 3532} 3533 3534static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft) 3535{ 3536 struct mem_cgroup *mem = mem_cgroup_from_cont(cont); 3537 u64 val; 3538 int type, name; 3539 3540 type = MEMFILE_TYPE(cft->private); 3541 name = MEMFILE_ATTR(cft->private); 3542 switch (type) { 3543 case _MEM: 3544 if (name == RES_USAGE) 3545 val = mem_cgroup_usage(mem, false); 3546 else 3547 val = res_counter_read_u64(&mem->res, name); 3548 break; 3549 case _MEMSWAP: 3550 if (name == RES_USAGE) 3551 val = mem_cgroup_usage(mem, true); 3552 else 3553 val = res_counter_read_u64(&mem->memsw, name); 3554 break; 3555 default: 3556 BUG(); 3557 break; 3558 } 3559 return val; 3560} 3561/* 3562 * The user of this function is... 3563 * RES_LIMIT. 3564 */ 3565static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft, 3566 const char *buffer) 3567{ 3568 struct mem_cgroup *memcg = mem_cgroup_from_cont(cont); 3569 int type, name; 3570 unsigned long long val; 3571 int ret; 3572 3573 type = MEMFILE_TYPE(cft->private); 3574 name = MEMFILE_ATTR(cft->private); 3575 switch (name) { 3576 case RES_LIMIT: 3577 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */ 3578 ret = -EINVAL; 3579 break; 3580 } 3581 /* This function does all necessary parse...reuse it */ 3582 ret = res_counter_memparse_write_strategy(buffer, &val); 3583 if (ret) 3584 break; 3585 if (type == _MEM) 3586 ret = mem_cgroup_resize_limit(memcg, val); 3587 else 3588 ret = mem_cgroup_resize_memsw_limit(memcg, val); 3589 break; 3590 case RES_SOFT_LIMIT: 3591 ret = res_counter_memparse_write_strategy(buffer, &val); 3592 if (ret) 3593 break; 3594 /* 3595 * For memsw, soft limits are hard to implement in terms 3596 * of semantics, for now, we support soft limits for 3597 * control without swap 3598 */ 3599 if (type == _MEM) 3600 ret = res_counter_set_soft_limit(&memcg->res, val); 3601 else 3602 ret = -EINVAL; 3603 break; 3604 default: 3605 ret = -EINVAL; /* should be BUG() ? */ 3606 break; 3607 } 3608 return ret; 3609} 3610 3611static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg, 3612 unsigned long long *mem_limit, unsigned long long *memsw_limit) 3613{ 3614 struct cgroup *cgroup; 3615 unsigned long long min_limit, min_memsw_limit, tmp; 3616 3617 min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT); 3618 min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 3619 cgroup = memcg->css.cgroup; 3620 if (!memcg->use_hierarchy) 3621 goto out; 3622 3623 while (cgroup->parent) { 3624 cgroup = cgroup->parent; 3625 memcg = mem_cgroup_from_cont(cgroup); 3626 if (!memcg->use_hierarchy) 3627 break; 3628 tmp = res_counter_read_u64(&memcg->res, RES_LIMIT); 3629 min_limit = min(min_limit, tmp); 3630 tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 3631 min_memsw_limit = min(min_memsw_limit, tmp); 3632 } 3633out: 3634 *mem_limit = min_limit; 3635 *memsw_limit = min_memsw_limit; 3636 return; 3637} 3638 3639static int mem_cgroup_reset(struct cgroup *cont, unsigned int event) 3640{ 3641 struct mem_cgroup *mem; 3642 int type, name; 3643 3644 mem = mem_cgroup_from_cont(cont); 3645 type = MEMFILE_TYPE(event); 3646 name = MEMFILE_ATTR(event); 3647 switch (name) { 3648 case RES_MAX_USAGE: 3649 if (type == _MEM) 3650 res_counter_reset_max(&mem->res); 3651 else 3652 res_counter_reset_max(&mem->memsw); 3653 break; 3654 case RES_FAILCNT: 3655 if (type == _MEM) 3656 res_counter_reset_failcnt(&mem->res); 3657 else 3658 res_counter_reset_failcnt(&mem->memsw); 3659 break; 3660 } 3661 3662 return 0; 3663} 3664 3665static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp, 3666 struct cftype *cft) 3667{ 3668 return mem_cgroup_from_cont(cgrp)->move_charge_at_immigrate; 3669} 3670 3671#ifdef CONFIG_MMU 3672static int mem_cgroup_move_charge_write(struct cgroup *cgrp, 3673 struct cftype *cft, u64 val) 3674{ 3675 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp); 3676 3677 if (val >= (1 << NR_MOVE_TYPE)) 3678 return -EINVAL; 3679 /* 3680 * We check this value several times in both in can_attach() and 3681 * attach(), so we need cgroup lock to prevent this value from being 3682 * inconsistent. 3683 */ 3684 cgroup_lock(); 3685 mem->move_charge_at_immigrate = val; 3686 cgroup_unlock(); 3687 3688 return 0; 3689} 3690#else 3691static int mem_cgroup_move_charge_write(struct cgroup *cgrp, 3692 struct cftype *cft, u64 val) 3693{ 3694 return -ENOSYS; 3695} 3696#endif 3697 3698 3699/* For read statistics */ 3700enum { 3701 MCS_CACHE, 3702 MCS_RSS, 3703 MCS_FILE_MAPPED, 3704 MCS_PGPGIN, 3705 MCS_PGPGOUT, 3706 MCS_SWAP, 3707 MCS_INACTIVE_ANON, 3708 MCS_ACTIVE_ANON, 3709 MCS_INACTIVE_FILE, 3710 MCS_ACTIVE_FILE, 3711 MCS_UNEVICTABLE, 3712 NR_MCS_STAT, 3713}; 3714 3715struct mcs_total_stat { 3716 s64 stat[NR_MCS_STAT]; 3717}; 3718 3719struct { 3720 char *local_name; 3721 char *total_name; 3722} memcg_stat_strings[NR_MCS_STAT] = { 3723 {"cache", "total_cache"}, 3724 {"rss", "total_rss"}, 3725 {"mapped_file", "total_mapped_file"}, 3726 {"pgpgin", "total_pgpgin"}, 3727 {"pgpgout", "total_pgpgout"}, 3728 {"swap", "total_swap"}, 3729 {"inactive_anon", "total_inactive_anon"}, 3730 {"active_anon", "total_active_anon"}, 3731 {"inactive_file", "total_inactive_file"}, 3732 {"active_file", "total_active_file"}, 3733 {"unevictable", "total_unevictable"} 3734}; 3735 3736 3737static void 3738mem_cgroup_get_local_stat(struct mem_cgroup *mem, struct mcs_total_stat *s) 3739{ 3740 s64 val; 3741 3742 /* per cpu stat */ 3743 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE); 3744 s->stat[MCS_CACHE] += val * PAGE_SIZE; 3745 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS); 3746 s->stat[MCS_RSS] += val * PAGE_SIZE; 3747 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_FILE_MAPPED); 3748 s->stat[MCS_FILE_MAPPED] += val * PAGE_SIZE; 3749 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGIN_COUNT); 3750 s->stat[MCS_PGPGIN] += val; 3751 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGOUT_COUNT); 3752 s->stat[MCS_PGPGOUT] += val; 3753 if (do_swap_account) { 3754 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_SWAPOUT); 3755 s->stat[MCS_SWAP] += val * PAGE_SIZE; 3756 } 3757 3758 /* per zone stat */ 3759 val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_ANON); 3760 s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE; 3761 val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_ANON); 3762 s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE; 3763 val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_FILE); 3764 s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE; 3765 val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_FILE); 3766 s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE; 3767 val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE); 3768 s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE; 3769} 3770 3771static void 3772mem_cgroup_get_total_stat(struct mem_cgroup *mem, struct mcs_total_stat *s) 3773{ 3774 struct mem_cgroup *iter; 3775 3776 for_each_mem_cgroup_tree(iter, mem) 3777 mem_cgroup_get_local_stat(iter, s); 3778} 3779 3780static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft, 3781 struct cgroup_map_cb *cb) 3782{ 3783 struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont); 3784 struct mcs_total_stat mystat; 3785 int i; 3786 3787 memset(&mystat, 0, sizeof(mystat)); 3788 mem_cgroup_get_local_stat(mem_cont, &mystat); 3789 3790 for (i = 0; i < NR_MCS_STAT; i++) { 3791 if (i == MCS_SWAP && !do_swap_account) 3792 continue; 3793 cb->fill(cb, memcg_stat_strings[i].local_name, mystat.stat[i]); 3794 } 3795 3796 /* Hierarchical information */ 3797 { 3798 unsigned long long limit, memsw_limit; 3799 memcg_get_hierarchical_limit(mem_cont, &limit, &memsw_limit); 3800 cb->fill(cb, "hierarchical_memory_limit", limit); 3801 if (do_swap_account) 3802 cb->fill(cb, "hierarchical_memsw_limit", memsw_limit); 3803 } 3804 3805 memset(&mystat, 0, sizeof(mystat)); 3806 mem_cgroup_get_total_stat(mem_cont, &mystat); 3807 for (i = 0; i < NR_MCS_STAT; i++) { 3808 if (i == MCS_SWAP && !do_swap_account) 3809 continue; 3810 cb->fill(cb, memcg_stat_strings[i].total_name, mystat.stat[i]); 3811 } 3812 3813#ifdef CONFIG_DEBUG_VM 3814 cb->fill(cb, "inactive_ratio", calc_inactive_ratio(mem_cont, NULL)); 3815 3816 { 3817 int nid, zid; 3818 struct mem_cgroup_per_zone *mz; 3819 unsigned long recent_rotated[2] = {0, 0}; 3820 unsigned long recent_scanned[2] = {0, 0}; 3821 3822 for_each_online_node(nid) 3823 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 3824 mz = mem_cgroup_zoneinfo(mem_cont, nid, zid); 3825 3826 recent_rotated[0] += 3827 mz->reclaim_stat.recent_rotated[0]; 3828 recent_rotated[1] += 3829 mz->reclaim_stat.recent_rotated[1]; 3830 recent_scanned[0] += 3831 mz->reclaim_stat.recent_scanned[0]; 3832 recent_scanned[1] += 3833 mz->reclaim_stat.recent_scanned[1]; 3834 } 3835 cb->fill(cb, "recent_rotated_anon", recent_rotated[0]); 3836 cb->fill(cb, "recent_rotated_file", recent_rotated[1]); 3837 cb->fill(cb, "recent_scanned_anon", recent_scanned[0]); 3838 cb->fill(cb, "recent_scanned_file", recent_scanned[1]); 3839 } 3840#endif 3841 3842 return 0; 3843} 3844 3845static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft) 3846{ 3847 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); 3848 3849 return get_swappiness(memcg); 3850} 3851 3852static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft, 3853 u64 val) 3854{ 3855 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); 3856 struct mem_cgroup *parent; 3857 3858 if (val > 100) 3859 return -EINVAL; 3860 3861 if (cgrp->parent == NULL) 3862 return -EINVAL; 3863 3864 parent = mem_cgroup_from_cont(cgrp->parent); 3865 3866 cgroup_lock(); 3867 3868 /* If under hierarchy, only empty-root can set this value */ 3869 if ((parent->use_hierarchy) || 3870 (memcg->use_hierarchy && !list_empty(&cgrp->children))) { 3871 cgroup_unlock(); 3872 return -EINVAL; 3873 } 3874 3875 spin_lock(&memcg->reclaim_param_lock); 3876 memcg->swappiness = val; 3877 spin_unlock(&memcg->reclaim_param_lock); 3878 3879 cgroup_unlock(); 3880 3881 return 0; 3882} 3883 3884static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap) 3885{ 3886 struct mem_cgroup_threshold_ary *t; 3887 u64 usage; 3888 int i; 3889 3890 rcu_read_lock(); 3891 if (!swap) 3892 t = rcu_dereference(memcg->thresholds.primary); 3893 else 3894 t = rcu_dereference(memcg->memsw_thresholds.primary); 3895 3896 if (!t) 3897 goto unlock; 3898 3899 usage = mem_cgroup_usage(memcg, swap); 3900 3901 /* 3902 * current_threshold points to threshold just below usage. 3903 * If it's not true, a threshold was crossed after last 3904 * call of __mem_cgroup_threshold(). 3905 */ 3906 i = t->current_threshold; 3907 3908 /* 3909 * Iterate backward over array of thresholds starting from 3910 * current_threshold and check if a threshold is crossed. 3911 * If none of thresholds below usage is crossed, we read 3912 * only one element of the array here. 3913 */ 3914 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--) 3915 eventfd_signal(t->entries[i].eventfd, 1); 3916 3917 /* i = current_threshold + 1 */ 3918 i++; 3919 3920 /* 3921 * Iterate forward over array of thresholds starting from 3922 * current_threshold+1 and check if a threshold is crossed. 3923 * If none of thresholds above usage is crossed, we read 3924 * only one element of the array here. 3925 */ 3926 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++) 3927 eventfd_signal(t->entries[i].eventfd, 1); 3928 3929 /* Update current_threshold */ 3930 t->current_threshold = i - 1; 3931unlock: 3932 rcu_read_unlock(); 3933} 3934 3935static void mem_cgroup_threshold(struct mem_cgroup *memcg) 3936{ 3937 while (memcg) { 3938 __mem_cgroup_threshold(memcg, false); 3939 if (do_swap_account) 3940 __mem_cgroup_threshold(memcg, true); 3941 3942 memcg = parent_mem_cgroup(memcg); 3943 } 3944} 3945 3946static int compare_thresholds(const void *a, const void *b) 3947{ 3948 const struct mem_cgroup_threshold *_a = a; 3949 const struct mem_cgroup_threshold *_b = b; 3950 3951 return _a->threshold - _b->threshold; 3952} 3953 3954static int mem_cgroup_oom_notify_cb(struct mem_cgroup *mem) 3955{ 3956 struct mem_cgroup_eventfd_list *ev; 3957 3958 list_for_each_entry(ev, &mem->oom_notify, list) 3959 eventfd_signal(ev->eventfd, 1); 3960 return 0; 3961} 3962 3963static void mem_cgroup_oom_notify(struct mem_cgroup *mem) 3964{ 3965 struct mem_cgroup *iter; 3966 3967 for_each_mem_cgroup_tree(iter, mem) 3968 mem_cgroup_oom_notify_cb(iter); 3969} 3970 3971static int mem_cgroup_usage_register_event(struct cgroup *cgrp, 3972 struct cftype *cft, struct eventfd_ctx *eventfd, const char *args) 3973{ 3974 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); 3975 struct mem_cgroup_thresholds *thresholds; 3976 struct mem_cgroup_threshold_ary *new; 3977 int type = MEMFILE_TYPE(cft->private); 3978 u64 threshold, usage; 3979 int i, size, ret; 3980 3981 ret = res_counter_memparse_write_strategy(args, &threshold); 3982 if (ret) 3983 return ret; 3984 3985 mutex_lock(&memcg->thresholds_lock); 3986 3987 if (type == _MEM) 3988 thresholds = &memcg->thresholds; 3989 else if (type == _MEMSWAP) 3990 thresholds = &memcg->memsw_thresholds; 3991 else 3992 BUG(); 3993 3994 usage = mem_cgroup_usage(memcg, type == _MEMSWAP); 3995 3996 /* Check if a threshold crossed before adding a new one */ 3997 if (thresholds->primary) 3998 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 3999 4000 size = thresholds->primary ? thresholds->primary->size + 1 : 1; 4001 4002 /* Allocate memory for new array of thresholds */ 4003 new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold), 4004 GFP_KERNEL); 4005 if (!new) { 4006 ret = -ENOMEM; 4007 goto unlock; 4008 } 4009 new->size = size; 4010 4011 /* Copy thresholds (if any) to new array */ 4012 if (thresholds->primary) { 4013 memcpy(new->entries, thresholds->primary->entries, (size - 1) * 4014 sizeof(struct mem_cgroup_threshold)); 4015 } 4016 4017 /* Add new threshold */ 4018 new->entries[size - 1].eventfd = eventfd; 4019 new->entries[size - 1].threshold = threshold; 4020 4021 /* Sort thresholds. Registering of new threshold isn't time-critical */ 4022 sort(new->entries, size, sizeof(struct mem_cgroup_threshold), 4023 compare_thresholds, NULL); 4024 4025 /* Find current threshold */ 4026 new->current_threshold = -1; 4027 for (i = 0; i < size; i++) { 4028 if (new->entries[i].threshold < usage) { 4029 /* 4030 * new->current_threshold will not be used until 4031 * rcu_assign_pointer(), so it's safe to increment 4032 * it here. 4033 */ 4034 ++new->current_threshold; 4035 } 4036 } 4037 4038 /* Free old spare buffer and save old primary buffer as spare */ 4039 kfree(thresholds->spare); 4040 thresholds->spare = thresholds->primary; 4041 4042 rcu_assign_pointer(thresholds->primary, new); 4043 4044 /* To be sure that nobody uses thresholds */ 4045 synchronize_rcu(); 4046 4047unlock: 4048 mutex_unlock(&memcg->thresholds_lock); 4049 4050 return ret; 4051} 4052 4053static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp, 4054 struct cftype *cft, struct eventfd_ctx *eventfd) 4055{ 4056 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); 4057 struct mem_cgroup_thresholds *thresholds; 4058 struct mem_cgroup_threshold_ary *new; 4059 int type = MEMFILE_TYPE(cft->private); 4060 u64 usage; 4061 int i, j, size; 4062 4063 mutex_lock(&memcg->thresholds_lock); 4064 if (type == _MEM) 4065 thresholds = &memcg->thresholds; 4066 else if (type == _MEMSWAP) 4067 thresholds = &memcg->memsw_thresholds; 4068 else 4069 BUG(); 4070 4071 /* 4072 * Something went wrong if we trying to unregister a threshold 4073 * if we don't have thresholds 4074 */ 4075 BUG_ON(!thresholds); 4076 4077 usage = mem_cgroup_usage(memcg, type == _MEMSWAP); 4078 4079 /* Check if a threshold crossed before removing */ 4080 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 4081 4082 /* Calculate new number of threshold */ 4083 size = 0; 4084 for (i = 0; i < thresholds->primary->size; i++) { 4085 if (thresholds->primary->entries[i].eventfd != eventfd) 4086 size++; 4087 } 4088 4089 new = thresholds->spare; 4090 4091 /* Set thresholds array to NULL if we don't have thresholds */ 4092 if (!size) { 4093 kfree(new); 4094 new = NULL; 4095 goto swap_buffers; 4096 } 4097 4098 new->size = size; 4099 4100 /* Copy thresholds and find current threshold */ 4101 new->current_threshold = -1; 4102 for (i = 0, j = 0; i < thresholds->primary->size; i++) { 4103 if (thresholds->primary->entries[i].eventfd == eventfd) 4104 continue; 4105 4106 new->entries[j] = thresholds->primary->entries[i]; 4107 if (new->entries[j].threshold < usage) { 4108 /* 4109 * new->current_threshold will not be used 4110 * until rcu_assign_pointer(), so it's safe to increment 4111 * it here. 4112 */ 4113 ++new->current_threshold; 4114 } 4115 j++; 4116 } 4117 4118swap_buffers: 4119 /* Swap primary and spare array */ 4120 thresholds->spare = thresholds->primary; 4121 rcu_assign_pointer(thresholds->primary, new); 4122 4123 /* To be sure that nobody uses thresholds */ 4124 synchronize_rcu(); 4125 4126 mutex_unlock(&memcg->thresholds_lock); 4127} 4128 4129static int mem_cgroup_oom_register_event(struct cgroup *cgrp, 4130 struct cftype *cft, struct eventfd_ctx *eventfd, const char *args) 4131{ 4132 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); 4133 struct mem_cgroup_eventfd_list *event; 4134 int type = MEMFILE_TYPE(cft->private); 4135 4136 BUG_ON(type != _OOM_TYPE); 4137 event = kmalloc(sizeof(*event), GFP_KERNEL); 4138 if (!event) 4139 return -ENOMEM; 4140 4141 mutex_lock(&memcg_oom_mutex); 4142 4143 event->eventfd = eventfd; 4144 list_add(&event->list, &memcg->oom_notify); 4145 4146 /* already in OOM ? */ 4147 if (atomic_read(&memcg->oom_lock)) 4148 eventfd_signal(eventfd, 1); 4149 mutex_unlock(&memcg_oom_mutex); 4150 4151 return 0; 4152} 4153 4154static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp, 4155 struct cftype *cft, struct eventfd_ctx *eventfd) 4156{ 4157 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp); 4158 struct mem_cgroup_eventfd_list *ev, *tmp; 4159 int type = MEMFILE_TYPE(cft->private); 4160 4161 BUG_ON(type != _OOM_TYPE); 4162 4163 mutex_lock(&memcg_oom_mutex); 4164 4165 list_for_each_entry_safe(ev, tmp, &mem->oom_notify, list) { 4166 if (ev->eventfd == eventfd) { 4167 list_del(&ev->list); 4168 kfree(ev); 4169 } 4170 } 4171 4172 mutex_unlock(&memcg_oom_mutex); 4173} 4174 4175static int mem_cgroup_oom_control_read(struct cgroup *cgrp, 4176 struct cftype *cft, struct cgroup_map_cb *cb) 4177{ 4178 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp); 4179 4180 cb->fill(cb, "oom_kill_disable", mem->oom_kill_disable); 4181 4182 if (atomic_read(&mem->oom_lock)) 4183 cb->fill(cb, "under_oom", 1); 4184 else 4185 cb->fill(cb, "under_oom", 0); 4186 return 0; 4187} 4188 4189static int mem_cgroup_oom_control_write(struct cgroup *cgrp, 4190 struct cftype *cft, u64 val) 4191{ 4192 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp); 4193 struct mem_cgroup *parent; 4194 4195 /* cannot set to root cgroup and only 0 and 1 are allowed */ 4196 if (!cgrp->parent || !((val == 0) || (val == 1))) 4197 return -EINVAL; 4198 4199 parent = mem_cgroup_from_cont(cgrp->parent); 4200 4201 cgroup_lock(); 4202 /* oom-kill-disable is a flag for subhierarchy. */ 4203 if ((parent->use_hierarchy) || 4204 (mem->use_hierarchy && !list_empty(&cgrp->children))) { 4205 cgroup_unlock(); 4206 return -EINVAL; 4207 } 4208 mem->oom_kill_disable = val; 4209 if (!val) 4210 memcg_oom_recover(mem); 4211 cgroup_unlock(); 4212 return 0; 4213} 4214 4215static struct cftype mem_cgroup_files[] = { 4216 { 4217 .name = "usage_in_bytes", 4218 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE), 4219 .read_u64 = mem_cgroup_read, 4220 .register_event = mem_cgroup_usage_register_event, 4221 .unregister_event = mem_cgroup_usage_unregister_event, 4222 }, 4223 { 4224 .name = "max_usage_in_bytes", 4225 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE), 4226 .trigger = mem_cgroup_reset, 4227 .read_u64 = mem_cgroup_read, 4228 }, 4229 { 4230 .name = "limit_in_bytes", 4231 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT), 4232 .write_string = mem_cgroup_write, 4233 .read_u64 = mem_cgroup_read, 4234 }, 4235 { 4236 .name = "soft_limit_in_bytes", 4237 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT), 4238 .write_string = mem_cgroup_write, 4239 .read_u64 = mem_cgroup_read, 4240 }, 4241 { 4242 .name = "failcnt", 4243 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT), 4244 .trigger = mem_cgroup_reset, 4245 .read_u64 = mem_cgroup_read, 4246 }, 4247 { 4248 .name = "stat", 4249 .read_map = mem_control_stat_show, 4250 }, 4251 { 4252 .name = "force_empty", 4253 .trigger = mem_cgroup_force_empty_write, 4254 }, 4255 { 4256 .name = "use_hierarchy", 4257 .write_u64 = mem_cgroup_hierarchy_write, 4258 .read_u64 = mem_cgroup_hierarchy_read, 4259 }, 4260 { 4261 .name = "swappiness", 4262 .read_u64 = mem_cgroup_swappiness_read, 4263 .write_u64 = mem_cgroup_swappiness_write, 4264 }, 4265 { 4266 .name = "move_charge_at_immigrate", 4267 .read_u64 = mem_cgroup_move_charge_read, 4268 .write_u64 = mem_cgroup_move_charge_write, 4269 }, 4270 { 4271 .name = "oom_control", 4272 .read_map = mem_cgroup_oom_control_read, 4273 .write_u64 = mem_cgroup_oom_control_write, 4274 .register_event = mem_cgroup_oom_register_event, 4275 .unregister_event = mem_cgroup_oom_unregister_event, 4276 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL), 4277 }, 4278}; 4279 4280#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP 4281static struct cftype memsw_cgroup_files[] = { 4282 { 4283 .name = "memsw.usage_in_bytes", 4284 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE), 4285 .read_u64 = mem_cgroup_read, 4286 .register_event = mem_cgroup_usage_register_event, 4287 .unregister_event = mem_cgroup_usage_unregister_event, 4288 }, 4289 { 4290 .name = "memsw.max_usage_in_bytes", 4291 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE), 4292 .trigger = mem_cgroup_reset, 4293 .read_u64 = mem_cgroup_read, 4294 }, 4295 { 4296 .name = "memsw.limit_in_bytes", 4297 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT), 4298 .write_string = mem_cgroup_write, 4299 .read_u64 = mem_cgroup_read, 4300 }, 4301 { 4302 .name = "memsw.failcnt", 4303 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT), 4304 .trigger = mem_cgroup_reset, 4305 .read_u64 = mem_cgroup_read, 4306 }, 4307}; 4308 4309static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss) 4310{ 4311 if (!do_swap_account) 4312 return 0; 4313 return cgroup_add_files(cont, ss, memsw_cgroup_files, 4314 ARRAY_SIZE(memsw_cgroup_files)); 4315}; 4316#else 4317static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss) 4318{ 4319 return 0; 4320} 4321#endif 4322 4323static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node) 4324{ 4325 struct mem_cgroup_per_node *pn; 4326 struct mem_cgroup_per_zone *mz; 4327 enum lru_list l; 4328 int zone, tmp = node; 4329 /* 4330 * This routine is called against possible nodes. 4331 * But it's BUG to call kmalloc() against offline node. 4332 * 4333 * TODO: this routine can waste much memory for nodes which will 4334 * never be onlined. It's better to use memory hotplug callback 4335 * function. 4336 */ 4337 if (!node_state(node, N_NORMAL_MEMORY)) 4338 tmp = -1; 4339 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp); 4340 if (!pn) 4341 return 1; 4342 4343 mem->info.nodeinfo[node] = pn; 4344 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 4345 mz = &pn->zoneinfo[zone]; 4346 for_each_lru(l) 4347 INIT_LIST_HEAD(&mz->lists[l]); 4348 mz->usage_in_excess = 0; 4349 mz->on_tree = false; 4350 mz->mem = mem; 4351 } 4352 return 0; 4353} 4354 4355static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node) 4356{ 4357 kfree(mem->info.nodeinfo[node]); 4358} 4359 4360static struct mem_cgroup *mem_cgroup_alloc(void) 4361{ 4362 struct mem_cgroup *mem; 4363 int size = sizeof(struct mem_cgroup); 4364 4365 /* Can be very big if MAX_NUMNODES is very big */ 4366 if (size < PAGE_SIZE) 4367 mem = kzalloc(size, GFP_KERNEL); 4368 else 4369 mem = vzalloc(size); 4370 4371 if (!mem) 4372 return NULL; 4373 4374 mem->stat = alloc_percpu(struct mem_cgroup_stat_cpu); 4375 if (!mem->stat) 4376 goto out_free; 4377 spin_lock_init(&mem->pcp_counter_lock); 4378 return mem; 4379 4380out_free: 4381 if (size < PAGE_SIZE) 4382 kfree(mem); 4383 else 4384 vfree(mem); 4385 return NULL; 4386} 4387 4388/* 4389 * At destroying mem_cgroup, references from swap_cgroup can remain. 4390 * (scanning all at force_empty is too costly...) 4391 * 4392 * Instead of clearing all references at force_empty, we remember 4393 * the number of reference from swap_cgroup and free mem_cgroup when 4394 * it goes down to 0. 4395 * 4396 * Removal of cgroup itself succeeds regardless of refs from swap. 4397 */ 4398 4399static void __mem_cgroup_free(struct mem_cgroup *mem) 4400{ 4401 int node; 4402 4403 mem_cgroup_remove_from_trees(mem); 4404 free_css_id(&mem_cgroup_subsys, &mem->css); 4405 4406 for_each_node_state(node, N_POSSIBLE) 4407 free_mem_cgroup_per_zone_info(mem, node); 4408 4409 free_percpu(mem->stat); 4410 if (sizeof(struct mem_cgroup) < PAGE_SIZE) 4411 kfree(mem); 4412 else 4413 vfree(mem); 4414} 4415 4416static void mem_cgroup_get(struct mem_cgroup *mem) 4417{ 4418 atomic_inc(&mem->refcnt); 4419} 4420 4421static void __mem_cgroup_put(struct mem_cgroup *mem, int count) 4422{ 4423 if (atomic_sub_and_test(count, &mem->refcnt)) { 4424 struct mem_cgroup *parent = parent_mem_cgroup(mem); 4425 __mem_cgroup_free(mem); 4426 if (parent) 4427 mem_cgroup_put(parent); 4428 } 4429} 4430 4431static void mem_cgroup_put(struct mem_cgroup *mem) 4432{ 4433 __mem_cgroup_put(mem, 1); 4434} 4435 4436/* 4437 * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled. 4438 */ 4439static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem) 4440{ 4441 if (!mem->res.parent) 4442 return NULL; 4443 return mem_cgroup_from_res_counter(mem->res.parent, res); 4444} 4445 4446#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP 4447static void __init enable_swap_cgroup(void) 4448{ 4449 if (!mem_cgroup_disabled() && really_do_swap_account) 4450 do_swap_account = 1; 4451} 4452#else 4453static void __init enable_swap_cgroup(void) 4454{ 4455} 4456#endif 4457 4458static int mem_cgroup_soft_limit_tree_init(void) 4459{ 4460 struct mem_cgroup_tree_per_node *rtpn; 4461 struct mem_cgroup_tree_per_zone *rtpz; 4462 int tmp, node, zone; 4463 4464 for_each_node_state(node, N_POSSIBLE) { 4465 tmp = node; 4466 if (!node_state(node, N_NORMAL_MEMORY)) 4467 tmp = -1; 4468 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp); 4469 if (!rtpn) 4470 return 1; 4471 4472 soft_limit_tree.rb_tree_per_node[node] = rtpn; 4473 4474 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 4475 rtpz = &rtpn->rb_tree_per_zone[zone]; 4476 rtpz->rb_root = RB_ROOT; 4477 spin_lock_init(&rtpz->lock); 4478 } 4479 } 4480 return 0; 4481} 4482 4483static struct cgroup_subsys_state * __ref 4484mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont) 4485{ 4486 struct mem_cgroup *mem, *parent; 4487 long error = -ENOMEM; 4488 int node; 4489 4490 mem = mem_cgroup_alloc(); 4491 if (!mem) 4492 return ERR_PTR(error); 4493 4494 for_each_node_state(node, N_POSSIBLE) 4495 if (alloc_mem_cgroup_per_zone_info(mem, node)) 4496 goto free_out; 4497 4498 /* root ? */ 4499 if (cont->parent == NULL) { 4500 int cpu; 4501 enable_swap_cgroup(); 4502 parent = NULL; 4503 root_mem_cgroup = mem; 4504 if (mem_cgroup_soft_limit_tree_init()) 4505 goto free_out; 4506 for_each_possible_cpu(cpu) { 4507 struct memcg_stock_pcp *stock = 4508 &per_cpu(memcg_stock, cpu); 4509 INIT_WORK(&stock->work, drain_local_stock); 4510 } 4511 hotcpu_notifier(memcg_cpu_hotplug_callback, 0); 4512 } else { 4513 parent = mem_cgroup_from_cont(cont->parent); 4514 mem->use_hierarchy = parent->use_hierarchy; 4515 mem->oom_kill_disable = parent->oom_kill_disable; 4516 } 4517 4518 if (parent && parent->use_hierarchy) { 4519 res_counter_init(&mem->res, &parent->res); 4520 res_counter_init(&mem->memsw, &parent->memsw); 4521 /* 4522 * We increment refcnt of the parent to ensure that we can 4523 * safely access it on res_counter_charge/uncharge. 4524 * This refcnt will be decremented when freeing this 4525 * mem_cgroup(see mem_cgroup_put). 4526 */ 4527 mem_cgroup_get(parent); 4528 } else { 4529 res_counter_init(&mem->res, NULL); 4530 res_counter_init(&mem->memsw, NULL); 4531 } 4532 mem->last_scanned_child = 0; 4533 spin_lock_init(&mem->reclaim_param_lock); 4534 INIT_LIST_HEAD(&mem->oom_notify); 4535 4536 if (parent) 4537 mem->swappiness = get_swappiness(parent); 4538 atomic_set(&mem->refcnt, 1); 4539 mem->move_charge_at_immigrate = 0; 4540 mutex_init(&mem->thresholds_lock); 4541 return &mem->css; 4542free_out: 4543 __mem_cgroup_free(mem); 4544 root_mem_cgroup = NULL; 4545 return ERR_PTR(error); 4546} 4547 4548static int mem_cgroup_pre_destroy(struct cgroup_subsys *ss, 4549 struct cgroup *cont) 4550{ 4551 struct mem_cgroup *mem = mem_cgroup_from_cont(cont); 4552 4553 return mem_cgroup_force_empty(mem, false); 4554} 4555 4556static void mem_cgroup_destroy(struct cgroup_subsys *ss, 4557 struct cgroup *cont) 4558{ 4559 struct mem_cgroup *mem = mem_cgroup_from_cont(cont); 4560 4561 mem_cgroup_put(mem); 4562} 4563 4564static int mem_cgroup_populate(struct cgroup_subsys *ss, 4565 struct cgroup *cont) 4566{ 4567 int ret; 4568 4569 ret = cgroup_add_files(cont, ss, mem_cgroup_files, 4570 ARRAY_SIZE(mem_cgroup_files)); 4571 4572 if (!ret) 4573 ret = register_memsw_files(cont, ss); 4574 return ret; 4575} 4576 4577#ifdef CONFIG_MMU 4578/* Handlers for move charge at task migration. */ 4579#define PRECHARGE_COUNT_AT_ONCE 256 4580static int mem_cgroup_do_precharge(unsigned long count) 4581{ 4582 int ret = 0; 4583 int batch_count = PRECHARGE_COUNT_AT_ONCE; 4584 struct mem_cgroup *mem = mc.to; 4585 4586 if (mem_cgroup_is_root(mem)) { 4587 mc.precharge += count; 4588 /* we don't need css_get for root */ 4589 return ret; 4590 } 4591 /* try to charge at once */ 4592 if (count > 1) { 4593 struct res_counter *dummy; 4594 /* 4595 * "mem" cannot be under rmdir() because we've already checked 4596 * by cgroup_lock_live_cgroup() that it is not removed and we 4597 * are still under the same cgroup_mutex. So we can postpone 4598 * css_get(). 4599 */ 4600 if (res_counter_charge(&mem->res, PAGE_SIZE * count, &dummy)) 4601 goto one_by_one; 4602 if (do_swap_account && res_counter_charge(&mem->memsw, 4603 PAGE_SIZE * count, &dummy)) { 4604 res_counter_uncharge(&mem->res, PAGE_SIZE * count); 4605 goto one_by_one; 4606 } 4607 mc.precharge += count; 4608 return ret; 4609 } 4610one_by_one: 4611 /* fall back to one by one charge */ 4612 while (count--) { 4613 if (signal_pending(current)) { 4614 ret = -EINTR; 4615 break; 4616 } 4617 if (!batch_count--) { 4618 batch_count = PRECHARGE_COUNT_AT_ONCE; 4619 cond_resched(); 4620 } 4621 ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem, false, 4622 PAGE_SIZE); 4623 if (ret || !mem) 4624 /* mem_cgroup_clear_mc() will do uncharge later */ 4625 return -ENOMEM; 4626 mc.precharge++; 4627 } 4628 return ret; 4629} 4630 4631/** 4632 * is_target_pte_for_mc - check a pte whether it is valid for move charge 4633 * @vma: the vma the pte to be checked belongs 4634 * @addr: the address corresponding to the pte to be checked 4635 * @ptent: the pte to be checked 4636 * @target: the pointer the target page or swap ent will be stored(can be NULL) 4637 * 4638 * Returns 4639 * 0(MC_TARGET_NONE): if the pte is not a target for move charge. 4640 * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for 4641 * move charge. if @target is not NULL, the page is stored in target->page 4642 * with extra refcnt got(Callers should handle it). 4643 * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a 4644 * target for charge migration. if @target is not NULL, the entry is stored 4645 * in target->ent. 4646 * 4647 * Called with pte lock held. 4648 */ 4649union mc_target { 4650 struct page *page; 4651 swp_entry_t ent; 4652}; 4653 4654enum mc_target_type { 4655 MC_TARGET_NONE, /* not used */ 4656 MC_TARGET_PAGE, 4657 MC_TARGET_SWAP, 4658}; 4659 4660static struct page *mc_handle_present_pte(struct vm_area_struct *vma, 4661 unsigned long addr, pte_t ptent) 4662{ 4663 struct page *page = vm_normal_page(vma, addr, ptent); 4664 4665 if (!page || !page_mapped(page)) 4666 return NULL; 4667 if (PageAnon(page)) { 4668 /* we don't move shared anon */ 4669 if (!move_anon() || page_mapcount(page) > 2) 4670 return NULL; 4671 } else if (!move_file()) 4672 /* we ignore mapcount for file pages */ 4673 return NULL; 4674 if (!get_page_unless_zero(page)) 4675 return NULL; 4676 4677 return page; 4678} 4679 4680static struct page *mc_handle_swap_pte(struct vm_area_struct *vma, 4681 unsigned long addr, pte_t ptent, swp_entry_t *entry) 4682{ 4683 int usage_count; 4684 struct page *page = NULL; 4685 swp_entry_t ent = pte_to_swp_entry(ptent); 4686 4687 if (!move_anon() || non_swap_entry(ent)) 4688 return NULL; 4689 usage_count = mem_cgroup_count_swap_user(ent, &page); 4690 if (usage_count > 1) { /* we don't move shared anon */ 4691 if (page) 4692 put_page(page); 4693 return NULL; 4694 } 4695 if (do_swap_account) 4696 entry->val = ent.val; 4697 4698 return page; 4699} 4700 4701static struct page *mc_handle_file_pte(struct vm_area_struct *vma, 4702 unsigned long addr, pte_t ptent, swp_entry_t *entry) 4703{ 4704 struct page *page = NULL; 4705 struct inode *inode; 4706 struct address_space *mapping; 4707 pgoff_t pgoff; 4708 4709 if (!vma->vm_file) /* anonymous vma */ 4710 return NULL; 4711 if (!move_file()) 4712 return NULL; 4713 4714 inode = vma->vm_file->f_path.dentry->d_inode; 4715 mapping = vma->vm_file->f_mapping; 4716 if (pte_none(ptent)) 4717 pgoff = linear_page_index(vma, addr); 4718 else /* pte_file(ptent) is true */ 4719 pgoff = pte_to_pgoff(ptent); 4720 4721 /* page is moved even if it's not RSS of this task(page-faulted). */ 4722 if (!mapping_cap_swap_backed(mapping)) { /* normal file */ 4723 page = find_get_page(mapping, pgoff); 4724 } else { /* shmem/tmpfs file. we should take account of swap too. */ 4725 swp_entry_t ent; 4726 mem_cgroup_get_shmem_target(inode, pgoff, &page, &ent); 4727 if (do_swap_account) 4728 entry->val = ent.val; 4729 } 4730 4731 return page; 4732} 4733 4734static int is_target_pte_for_mc(struct vm_area_struct *vma, 4735 unsigned long addr, pte_t ptent, union mc_target *target) 4736{ 4737 struct page *page = NULL; 4738 struct page_cgroup *pc; 4739 int ret = 0; 4740 swp_entry_t ent = { .val = 0 }; 4741 4742 if (pte_present(ptent)) 4743 page = mc_handle_present_pte(vma, addr, ptent); 4744 else if (is_swap_pte(ptent)) 4745 page = mc_handle_swap_pte(vma, addr, ptent, &ent); 4746 else if (pte_none(ptent) || pte_file(ptent)) 4747 page = mc_handle_file_pte(vma, addr, ptent, &ent); 4748 4749 if (!page && !ent.val) 4750 return 0; 4751 if (page) { 4752 pc = lookup_page_cgroup(page); 4753 /* 4754 * Do only loose check w/o page_cgroup lock. 4755 * mem_cgroup_move_account() checks the pc is valid or not under 4756 * the lock. 4757 */ 4758 if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) { 4759 ret = MC_TARGET_PAGE; 4760 if (target) 4761 target->page = page; 4762 } 4763 if (!ret || !target) 4764 put_page(page); 4765 } 4766 /* There is a swap entry and a page doesn't exist or isn't charged */ 4767 if (ent.val && !ret && 4768 css_id(&mc.from->css) == lookup_swap_cgroup(ent)) { 4769 ret = MC_TARGET_SWAP; 4770 if (target) 4771 target->ent = ent; 4772 } 4773 return ret; 4774} 4775 4776static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd, 4777 unsigned long addr, unsigned long end, 4778 struct mm_walk *walk) 4779{ 4780 struct vm_area_struct *vma = walk->private; 4781 pte_t *pte; 4782 spinlock_t *ptl; 4783 4784 split_huge_page_pmd(walk->mm, pmd); 4785 4786 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 4787 for (; addr != end; pte++, addr += PAGE_SIZE) 4788 if (is_target_pte_for_mc(vma, addr, *pte, NULL)) 4789 mc.precharge++; /* increment precharge temporarily */ 4790 pte_unmap_unlock(pte - 1, ptl); 4791 cond_resched(); 4792 4793 return 0; 4794} 4795 4796static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm) 4797{ 4798 unsigned long precharge; 4799 struct vm_area_struct *vma; 4800 4801 down_read(&mm->mmap_sem); 4802 for (vma = mm->mmap; vma; vma = vma->vm_next) { 4803 struct mm_walk mem_cgroup_count_precharge_walk = { 4804 .pmd_entry = mem_cgroup_count_precharge_pte_range, 4805 .mm = mm, 4806 .private = vma, 4807 }; 4808 if (is_vm_hugetlb_page(vma)) 4809 continue; 4810 walk_page_range(vma->vm_start, vma->vm_end, 4811 &mem_cgroup_count_precharge_walk); 4812 } 4813 up_read(&mm->mmap_sem); 4814 4815 precharge = mc.precharge; 4816 mc.precharge = 0; 4817 4818 return precharge; 4819} 4820 4821static int mem_cgroup_precharge_mc(struct mm_struct *mm) 4822{ 4823 unsigned long precharge = mem_cgroup_count_precharge(mm); 4824 4825 VM_BUG_ON(mc.moving_task); 4826 mc.moving_task = current; 4827 return mem_cgroup_do_precharge(precharge); 4828} 4829 4830/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */ 4831static void __mem_cgroup_clear_mc(void) 4832{ 4833 struct mem_cgroup *from = mc.from; 4834 struct mem_cgroup *to = mc.to; 4835 4836 /* we must uncharge all the leftover precharges from mc.to */ 4837 if (mc.precharge) { 4838 __mem_cgroup_cancel_charge(mc.to, mc.precharge); 4839 mc.precharge = 0; 4840 } 4841 /* 4842 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so 4843 * we must uncharge here. 4844 */ 4845 if (mc.moved_charge) { 4846 __mem_cgroup_cancel_charge(mc.from, mc.moved_charge); 4847 mc.moved_charge = 0; 4848 } 4849 /* we must fixup refcnts and charges */ 4850 if (mc.moved_swap) { 4851 /* uncharge swap account from the old cgroup */ 4852 if (!mem_cgroup_is_root(mc.from)) 4853 res_counter_uncharge(&mc.from->memsw, 4854 PAGE_SIZE * mc.moved_swap); 4855 __mem_cgroup_put(mc.from, mc.moved_swap); 4856 4857 if (!mem_cgroup_is_root(mc.to)) { 4858 /* 4859 * we charged both to->res and to->memsw, so we should 4860 * uncharge to->res. 4861 */ 4862 res_counter_uncharge(&mc.to->res, 4863 PAGE_SIZE * mc.moved_swap); 4864 } 4865 /* we've already done mem_cgroup_get(mc.to) */ 4866 mc.moved_swap = 0; 4867 } 4868 memcg_oom_recover(from); 4869 memcg_oom_recover(to); 4870 wake_up_all(&mc.waitq); 4871} 4872 4873static void mem_cgroup_clear_mc(void) 4874{ 4875 struct mem_cgroup *from = mc.from; 4876 4877 /* 4878 * we must clear moving_task before waking up waiters at the end of 4879 * task migration. 4880 */ 4881 mc.moving_task = NULL; 4882 __mem_cgroup_clear_mc(); 4883 spin_lock(&mc.lock); 4884 mc.from = NULL; 4885 mc.to = NULL; 4886 spin_unlock(&mc.lock); 4887 mem_cgroup_end_move(from); 4888} 4889 4890static int mem_cgroup_can_attach(struct cgroup_subsys *ss, 4891 struct cgroup *cgroup, 4892 struct task_struct *p, 4893 bool threadgroup) 4894{ 4895 int ret = 0; 4896 struct mem_cgroup *mem = mem_cgroup_from_cont(cgroup); 4897 4898 if (mem->move_charge_at_immigrate) { 4899 struct mm_struct *mm; 4900 struct mem_cgroup *from = mem_cgroup_from_task(p); 4901 4902 VM_BUG_ON(from == mem); 4903 4904 mm = get_task_mm(p); 4905 if (!mm) 4906 return 0; 4907 /* We move charges only when we move a owner of the mm */ 4908 if (mm->owner == p) { 4909 VM_BUG_ON(mc.from); 4910 VM_BUG_ON(mc.to); 4911 VM_BUG_ON(mc.precharge); 4912 VM_BUG_ON(mc.moved_charge); 4913 VM_BUG_ON(mc.moved_swap); 4914 mem_cgroup_start_move(from); 4915 spin_lock(&mc.lock); 4916 mc.from = from; 4917 mc.to = mem; 4918 spin_unlock(&mc.lock); 4919 /* We set mc.moving_task later */ 4920 4921 ret = mem_cgroup_precharge_mc(mm); 4922 if (ret) 4923 mem_cgroup_clear_mc(); 4924 } 4925 mmput(mm); 4926 } 4927 return ret; 4928} 4929 4930static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss, 4931 struct cgroup *cgroup, 4932 struct task_struct *p, 4933 bool threadgroup) 4934{ 4935 mem_cgroup_clear_mc(); 4936} 4937 4938static int mem_cgroup_move_charge_pte_range(pmd_t *pmd, 4939 unsigned long addr, unsigned long end, 4940 struct mm_walk *walk) 4941{ 4942 int ret = 0; 4943 struct vm_area_struct *vma = walk->private; 4944 pte_t *pte; 4945 spinlock_t *ptl; 4946 4947 split_huge_page_pmd(walk->mm, pmd); 4948retry: 4949 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 4950 for (; addr != end; addr += PAGE_SIZE) { 4951 pte_t ptent = *(pte++); 4952 union mc_target target; 4953 int type; 4954 struct page *page; 4955 struct page_cgroup *pc; 4956 swp_entry_t ent; 4957 4958 if (!mc.precharge) 4959 break; 4960 4961 type = is_target_pte_for_mc(vma, addr, ptent, &target); 4962 switch (type) { 4963 case MC_TARGET_PAGE: 4964 page = target.page; 4965 if (isolate_lru_page(page)) 4966 goto put; 4967 pc = lookup_page_cgroup(page); 4968 if (!mem_cgroup_move_account(pc, 4969 mc.from, mc.to, false, PAGE_SIZE)) { 4970 mc.precharge--; 4971 /* we uncharge from mc.from later. */ 4972 mc.moved_charge++; 4973 } 4974 putback_lru_page(page); 4975put: /* is_target_pte_for_mc() gets the page */ 4976 put_page(page); 4977 break; 4978 case MC_TARGET_SWAP: 4979 ent = target.ent; 4980 if (!mem_cgroup_move_swap_account(ent, 4981 mc.from, mc.to, false)) { 4982 mc.precharge--; 4983 /* we fixup refcnts and charges later. */ 4984 mc.moved_swap++; 4985 } 4986 break; 4987 default: 4988 break; 4989 } 4990 } 4991 pte_unmap_unlock(pte - 1, ptl); 4992 cond_resched(); 4993 4994 if (addr != end) { 4995 /* 4996 * We have consumed all precharges we got in can_attach(). 4997 * We try charge one by one, but don't do any additional 4998 * charges to mc.to if we have failed in charge once in attach() 4999 * phase. 5000 */ 5001 ret = mem_cgroup_do_precharge(1); 5002 if (!ret) 5003 goto retry; 5004 } 5005 5006 return ret; 5007} 5008 5009static void mem_cgroup_move_charge(struct mm_struct *mm) 5010{ 5011 struct vm_area_struct *vma; 5012 5013 lru_add_drain_all(); 5014retry: 5015 if (unlikely(!down_read_trylock(&mm->mmap_sem))) { 5016 /* 5017 * Someone who are holding the mmap_sem might be waiting in 5018 * waitq. So we cancel all extra charges, wake up all waiters, 5019 * and retry. Because we cancel precharges, we might not be able 5020 * to move enough charges, but moving charge is a best-effort 5021 * feature anyway, so it wouldn't be a big problem. 5022 */ 5023 __mem_cgroup_clear_mc(); 5024 cond_resched(); 5025 goto retry; 5026 } 5027 for (vma = mm->mmap; vma; vma = vma->vm_next) { 5028 int ret; 5029 struct mm_walk mem_cgroup_move_charge_walk = { 5030 .pmd_entry = mem_cgroup_move_charge_pte_range, 5031 .mm = mm, 5032 .private = vma, 5033 }; 5034 if (is_vm_hugetlb_page(vma)) 5035 continue; 5036 ret = walk_page_range(vma->vm_start, vma->vm_end, 5037 &mem_cgroup_move_charge_walk); 5038 if (ret) 5039 /* 5040 * means we have consumed all precharges and failed in 5041 * doing additional charge. Just abandon here. 5042 */ 5043 break; 5044 } 5045 up_read(&mm->mmap_sem); 5046} 5047 5048static void mem_cgroup_move_task(struct cgroup_subsys *ss, 5049 struct cgroup *cont, 5050 struct cgroup *old_cont, 5051 struct task_struct *p, 5052 bool threadgroup) 5053{ 5054 struct mm_struct *mm; 5055 5056 if (!mc.to) 5057 /* no need to move charge */ 5058 return; 5059 5060 mm = get_task_mm(p); 5061 if (mm) { 5062 mem_cgroup_move_charge(mm); 5063 mmput(mm); 5064 } 5065 mem_cgroup_clear_mc(); 5066} 5067#else /* !CONFIG_MMU */ 5068static int mem_cgroup_can_attach(struct cgroup_subsys *ss, 5069 struct cgroup *cgroup, 5070 struct task_struct *p, 5071 bool threadgroup) 5072{ 5073 return 0; 5074} 5075static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss, 5076 struct cgroup *cgroup, 5077 struct task_struct *p, 5078 bool threadgroup) 5079{ 5080} 5081static void mem_cgroup_move_task(struct cgroup_subsys *ss, 5082 struct cgroup *cont, 5083 struct cgroup *old_cont, 5084 struct task_struct *p, 5085 bool threadgroup) 5086{ 5087} 5088#endif 5089 5090struct cgroup_subsys mem_cgroup_subsys = { 5091 .name = "memory", 5092 .subsys_id = mem_cgroup_subsys_id, 5093 .create = mem_cgroup_create, 5094 .pre_destroy = mem_cgroup_pre_destroy, 5095 .destroy = mem_cgroup_destroy, 5096 .populate = mem_cgroup_populate, 5097 .can_attach = mem_cgroup_can_attach, 5098 .cancel_attach = mem_cgroup_cancel_attach, 5099 .attach = mem_cgroup_move_task, 5100 .early_init = 0, 5101 .use_id = 1, 5102}; 5103 5104#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP 5105static int __init enable_swap_account(char *s) 5106{ 5107 /* consider enabled if no parameter or 1 is given */ 5108 if (!(*s) || !strcmp(s, "=1")) 5109 really_do_swap_account = 1; 5110 else if (!strcmp(s, "=0")) 5111 really_do_swap_account = 0; 5112 return 1; 5113} 5114__setup("swapaccount", enable_swap_account); 5115 5116static int __init disable_swap_account(char *s) 5117{ 5118 printk_once("noswapaccount is deprecated and will be removed in 2.6.40. Use swapaccount=0 instead\n"); 5119 enable_swap_account("=0"); 5120 return 1; 5121} 5122__setup("noswapaccount", disable_swap_account); 5123#endif 5124