1 /* memcontrol.c - Memory Controller 2 * 3 * Copyright IBM Corporation, 2007 4 * Author Balbir Singh <balbir@linux.vnet.ibm.com> 5 * 6 * Copyright 2007 OpenVZ SWsoft Inc 7 * Author: Pavel Emelianov <xemul@openvz.org> 8 * 9 * Memory thresholds 10 * Copyright (C) 2009 Nokia Corporation 11 * Author: Kirill A. Shutemov 12 * 13 * This program is free software; you can redistribute it and/or modify 14 * it under the terms of the GNU General Public License as published by 15 * the Free Software Foundation; either version 2 of the License, or 16 * (at your option) any later version. 17 * 18 * This program is distributed in the hope that it will be useful, 19 * but WITHOUT ANY WARRANTY; without even the implied warranty of 20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 21 * GNU General Public License for more details. 22 */ 23 24 #include <linux/res_counter.h> 25 #include <linux/memcontrol.h> 26 #include <linux/cgroup.h> 27 #include <linux/mm.h> 28 #include <linux/hugetlb.h> 29 #include <linux/pagemap.h> 30 #include <linux/smp.h> 31 #include <linux/page-flags.h> 32 #include <linux/backing-dev.h> 33 #include <linux/bit_spinlock.h> 34 #include <linux/rcupdate.h> 35 #include <linux/limits.h> 36 #include <linux/mutex.h> 37 #include <linux/rbtree.h> 38 #include <linux/slab.h> 39 #include <linux/swap.h> 40 #include <linux/swapops.h> 41 #include <linux/spinlock.h> 42 #include <linux/eventfd.h> 43 #include <linux/sort.h> 44 #include <linux/fs.h> 45 #include <linux/seq_file.h> 46 #include <linux/vmalloc.h> 47 #include <linux/mm_inline.h> 48 #include <linux/page_cgroup.h> 49 #include <linux/cpu.h> 50 #include <linux/oom.h> 51 #include "internal.h" 52 53 #include <asm/uaccess.h> 54 55 #include <trace/events/vmscan.h> 56 57 struct cgroup_subsys mem_cgroup_subsys __read_mostly; 58 #define MEM_CGROUP_RECLAIM_RETRIES 5 59 struct mem_cgroup *root_mem_cgroup __read_mostly; 60 61 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP 62 /* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */ 63 int do_swap_account __read_mostly; 64 65 /* for remember boot option*/ 66 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP_ENABLED 67 static int really_do_swap_account __initdata = 1; 68 #else 69 static int really_do_swap_account __initdata = 0; 70 #endif 71 72 #else 73 #define do_swap_account (0) 74 #endif 75 76 /* 77 * Per memcg event counter is incremented at every pagein/pageout. This counter 78 * is used for trigger some periodic events. This is straightforward and better 79 * than using jiffies etc. to handle periodic memcg event. 80 * 81 * These values will be used as !((event) & ((1 <<(thresh)) - 1)) 82 */ 83 #define THRESHOLDS_EVENTS_THRESH (7) /* once in 128 */ 84 #define SOFTLIMIT_EVENTS_THRESH (10) /* once in 1024 */ 85 86 /* 87 * Statistics for memory cgroup. 88 */ 89 enum mem_cgroup_stat_index { 90 /* 91 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss. 92 */ 93 MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */ 94 MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */ 95 MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */ 96 MEM_CGROUP_STAT_PGPGIN_COUNT, /* # of pages paged in */ 97 MEM_CGROUP_STAT_PGPGOUT_COUNT, /* # of pages paged out */ 98 MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */ 99 MEM_CGROUP_STAT_DATA, /* end of data requires synchronization */ 100 /* incremented at every pagein/pageout */ 101 MEM_CGROUP_EVENTS = MEM_CGROUP_STAT_DATA, 102 MEM_CGROUP_ON_MOVE, /* someone is moving account between groups */ 103 104 MEM_CGROUP_STAT_NSTATS, 105 }; 106 107 struct mem_cgroup_stat_cpu { 108 s64 count[MEM_CGROUP_STAT_NSTATS]; 109 }; 110 111 /* 112 * per-zone information in memory controller. 113 */ 114 struct mem_cgroup_per_zone { 115 /* 116 * spin_lock to protect the per cgroup LRU 117 */ 118 struct list_head lists[NR_LRU_LISTS]; 119 unsigned long count[NR_LRU_LISTS]; 120 121 struct zone_reclaim_stat reclaim_stat; 122 struct rb_node tree_node; /* RB tree node */ 123 unsigned long long usage_in_excess;/* Set to the value by which */ 124 /* the soft limit is exceeded*/ 125 bool on_tree; 126 struct mem_cgroup *mem; /* Back pointer, we cannot */ 127 /* use container_of */ 128 }; 129 /* Macro for accessing counter */ 130 #define MEM_CGROUP_ZSTAT(mz, idx) ((mz)->count[(idx)]) 131 132 struct mem_cgroup_per_node { 133 struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES]; 134 }; 135 136 struct mem_cgroup_lru_info { 137 struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES]; 138 }; 139 140 /* 141 * Cgroups above their limits are maintained in a RB-Tree, independent of 142 * their hierarchy representation 143 */ 144 145 struct mem_cgroup_tree_per_zone { 146 struct rb_root rb_root; 147 spinlock_t lock; 148 }; 149 150 struct mem_cgroup_tree_per_node { 151 struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES]; 152 }; 153 154 struct mem_cgroup_tree { 155 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES]; 156 }; 157 158 static struct mem_cgroup_tree soft_limit_tree __read_mostly; 159 160 struct mem_cgroup_threshold { 161 struct eventfd_ctx *eventfd; 162 u64 threshold; 163 }; 164 165 /* For threshold */ 166 struct mem_cgroup_threshold_ary { 167 /* An array index points to threshold just below usage. */ 168 int current_threshold; 169 /* Size of entries[] */ 170 unsigned int size; 171 /* Array of thresholds */ 172 struct mem_cgroup_threshold entries[0]; 173 }; 174 175 struct mem_cgroup_thresholds { 176 /* Primary thresholds array */ 177 struct mem_cgroup_threshold_ary *primary; 178 /* 179 * Spare threshold array. 180 * This is needed to make mem_cgroup_unregister_event() "never fail". 181 * It must be able to store at least primary->size - 1 entries. 182 */ 183 struct mem_cgroup_threshold_ary *spare; 184 }; 185 186 /* for OOM */ 187 struct mem_cgroup_eventfd_list { 188 struct list_head list; 189 struct eventfd_ctx *eventfd; 190 }; 191 192 static void mem_cgroup_threshold(struct mem_cgroup *mem); 193 static void mem_cgroup_oom_notify(struct mem_cgroup *mem); 194 195 /* 196 * The memory controller data structure. The memory controller controls both 197 * page cache and RSS per cgroup. We would eventually like to provide 198 * statistics based on the statistics developed by Rik Van Riel for clock-pro, 199 * to help the administrator determine what knobs to tune. 200 * 201 * TODO: Add a water mark for the memory controller. Reclaim will begin when 202 * we hit the water mark. May be even add a low water mark, such that 203 * no reclaim occurs from a cgroup at it's low water mark, this is 204 * a feature that will be implemented much later in the future. 205 */ 206 struct mem_cgroup { 207 struct cgroup_subsys_state css; 208 /* 209 * the counter to account for memory usage 210 */ 211 struct res_counter res; 212 /* 213 * the counter to account for mem+swap usage. 214 */ 215 struct res_counter memsw; 216 /* 217 * Per cgroup active and inactive list, similar to the 218 * per zone LRU lists. 219 */ 220 struct mem_cgroup_lru_info info; 221 222 /* 223 protect against reclaim related member. 224 */ 225 spinlock_t reclaim_param_lock; 226 227 /* 228 * While reclaiming in a hierarchy, we cache the last child we 229 * reclaimed from. 230 */ 231 int last_scanned_child; 232 /* 233 * Should the accounting and control be hierarchical, per subtree? 234 */ 235 bool use_hierarchy; 236 atomic_t oom_lock; 237 atomic_t refcnt; 238 239 unsigned int swappiness; 240 /* OOM-Killer disable */ 241 int oom_kill_disable; 242 243 /* set when res.limit == memsw.limit */ 244 bool memsw_is_minimum; 245 246 /* protect arrays of thresholds */ 247 struct mutex thresholds_lock; 248 249 /* thresholds for memory usage. RCU-protected */ 250 struct mem_cgroup_thresholds thresholds; 251 252 /* thresholds for mem+swap usage. RCU-protected */ 253 struct mem_cgroup_thresholds memsw_thresholds; 254 255 /* For oom notifier event fd */ 256 struct list_head oom_notify; 257 258 /* 259 * Should we move charges of a task when a task is moved into this 260 * mem_cgroup ? And what type of charges should we move ? 261 */ 262 unsigned long move_charge_at_immigrate; 263 /* 264 * percpu counter. 265 */ 266 struct mem_cgroup_stat_cpu *stat; 267 /* 268 * used when a cpu is offlined or other synchronizations 269 * See mem_cgroup_read_stat(). 270 */ 271 struct mem_cgroup_stat_cpu nocpu_base; 272 spinlock_t pcp_counter_lock; 273 }; 274 275 /* Stuffs for move charges at task migration. */ 276 /* 277 * Types of charges to be moved. "move_charge_at_immitgrate" is treated as a 278 * left-shifted bitmap of these types. 279 */ 280 enum move_type { 281 MOVE_CHARGE_TYPE_ANON, /* private anonymous page and swap of it */ 282 MOVE_CHARGE_TYPE_FILE, /* file page(including tmpfs) and swap of it */ 283 NR_MOVE_TYPE, 284 }; 285 286 /* "mc" and its members are protected by cgroup_mutex */ 287 static struct move_charge_struct { 288 spinlock_t lock; /* for from, to */ 289 struct mem_cgroup *from; 290 struct mem_cgroup *to; 291 unsigned long precharge; 292 unsigned long moved_charge; 293 unsigned long moved_swap; 294 struct task_struct *moving_task; /* a task moving charges */ 295 wait_queue_head_t waitq; /* a waitq for other context */ 296 } mc = { 297 .lock = __SPIN_LOCK_UNLOCKED(mc.lock), 298 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq), 299 }; 300 301 static bool move_anon(void) 302 { 303 return test_bit(MOVE_CHARGE_TYPE_ANON, 304 &mc.to->move_charge_at_immigrate); 305 } 306 307 static bool move_file(void) 308 { 309 return test_bit(MOVE_CHARGE_TYPE_FILE, 310 &mc.to->move_charge_at_immigrate); 311 } 312 313 /* 314 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft 315 * limit reclaim to prevent infinite loops, if they ever occur. 316 */ 317 #define MEM_CGROUP_MAX_RECLAIM_LOOPS (100) 318 #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS (2) 319 320 enum charge_type { 321 MEM_CGROUP_CHARGE_TYPE_CACHE = 0, 322 MEM_CGROUP_CHARGE_TYPE_MAPPED, 323 MEM_CGROUP_CHARGE_TYPE_SHMEM, /* used by page migration of shmem */ 324 MEM_CGROUP_CHARGE_TYPE_FORCE, /* used by force_empty */ 325 MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */ 326 MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */ 327 NR_CHARGE_TYPE, 328 }; 329 330 /* only for here (for easy reading.) */ 331 #define PCGF_CACHE (1UL << PCG_CACHE) 332 #define PCGF_USED (1UL << PCG_USED) 333 #define PCGF_LOCK (1UL << PCG_LOCK) 334 /* Not used, but added here for completeness */ 335 #define PCGF_ACCT (1UL << PCG_ACCT) 336 337 /* for encoding cft->private value on file */ 338 #define _MEM (0) 339 #define _MEMSWAP (1) 340 #define _OOM_TYPE (2) 341 #define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val)) 342 #define MEMFILE_TYPE(val) (((val) >> 16) & 0xffff) 343 #define MEMFILE_ATTR(val) ((val) & 0xffff) 344 /* Used for OOM nofiier */ 345 #define OOM_CONTROL (0) 346 347 /* 348 * Reclaim flags for mem_cgroup_hierarchical_reclaim 349 */ 350 #define MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0 351 #define MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT) 352 #define MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1 353 #define MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT) 354 #define MEM_CGROUP_RECLAIM_SOFT_BIT 0x2 355 #define MEM_CGROUP_RECLAIM_SOFT (1 << MEM_CGROUP_RECLAIM_SOFT_BIT) 356 357 static void mem_cgroup_get(struct mem_cgroup *mem); 358 static void mem_cgroup_put(struct mem_cgroup *mem); 359 static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem); 360 static void drain_all_stock_async(void); 361 362 static struct mem_cgroup_per_zone * 363 mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid) 364 { 365 return &mem->info.nodeinfo[nid]->zoneinfo[zid]; 366 } 367 368 struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem) 369 { 370 return &mem->css; 371 } 372 373 static struct mem_cgroup_per_zone * 374 page_cgroup_zoneinfo(struct page_cgroup *pc) 375 { 376 struct mem_cgroup *mem = pc->mem_cgroup; 377 int nid = page_cgroup_nid(pc); 378 int zid = page_cgroup_zid(pc); 379 380 if (!mem) 381 return NULL; 382 383 return mem_cgroup_zoneinfo(mem, nid, zid); 384 } 385 386 static struct mem_cgroup_tree_per_zone * 387 soft_limit_tree_node_zone(int nid, int zid) 388 { 389 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid]; 390 } 391 392 static struct mem_cgroup_tree_per_zone * 393 soft_limit_tree_from_page(struct page *page) 394 { 395 int nid = page_to_nid(page); 396 int zid = page_zonenum(page); 397 398 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid]; 399 } 400 401 static void 402 __mem_cgroup_insert_exceeded(struct mem_cgroup *mem, 403 struct mem_cgroup_per_zone *mz, 404 struct mem_cgroup_tree_per_zone *mctz, 405 unsigned long long new_usage_in_excess) 406 { 407 struct rb_node **p = &mctz->rb_root.rb_node; 408 struct rb_node *parent = NULL; 409 struct mem_cgroup_per_zone *mz_node; 410 411 if (mz->on_tree) 412 return; 413 414 mz->usage_in_excess = new_usage_in_excess; 415 if (!mz->usage_in_excess) 416 return; 417 while (*p) { 418 parent = *p; 419 mz_node = rb_entry(parent, struct mem_cgroup_per_zone, 420 tree_node); 421 if (mz->usage_in_excess < mz_node->usage_in_excess) 422 p = &(*p)->rb_left; 423 /* 424 * We can't avoid mem cgroups that are over their soft 425 * limit by the same amount 426 */ 427 else if (mz->usage_in_excess >= mz_node->usage_in_excess) 428 p = &(*p)->rb_right; 429 } 430 rb_link_node(&mz->tree_node, parent, p); 431 rb_insert_color(&mz->tree_node, &mctz->rb_root); 432 mz->on_tree = true; 433 } 434 435 static void 436 __mem_cgroup_remove_exceeded(struct mem_cgroup *mem, 437 struct mem_cgroup_per_zone *mz, 438 struct mem_cgroup_tree_per_zone *mctz) 439 { 440 if (!mz->on_tree) 441 return; 442 rb_erase(&mz->tree_node, &mctz->rb_root); 443 mz->on_tree = false; 444 } 445 446 static void 447 mem_cgroup_remove_exceeded(struct mem_cgroup *mem, 448 struct mem_cgroup_per_zone *mz, 449 struct mem_cgroup_tree_per_zone *mctz) 450 { 451 spin_lock(&mctz->lock); 452 __mem_cgroup_remove_exceeded(mem, mz, mctz); 453 spin_unlock(&mctz->lock); 454 } 455 456 457 static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page) 458 { 459 unsigned long long excess; 460 struct mem_cgroup_per_zone *mz; 461 struct mem_cgroup_tree_per_zone *mctz; 462 int nid = page_to_nid(page); 463 int zid = page_zonenum(page); 464 mctz = soft_limit_tree_from_page(page); 465 466 /* 467 * Necessary to update all ancestors when hierarchy is used. 468 * because their event counter is not touched. 469 */ 470 for (; mem; mem = parent_mem_cgroup(mem)) { 471 mz = mem_cgroup_zoneinfo(mem, nid, zid); 472 excess = res_counter_soft_limit_excess(&mem->res); 473 /* 474 * We have to update the tree if mz is on RB-tree or 475 * mem is over its softlimit. 476 */ 477 if (excess || mz->on_tree) { 478 spin_lock(&mctz->lock); 479 /* if on-tree, remove it */ 480 if (mz->on_tree) 481 __mem_cgroup_remove_exceeded(mem, mz, mctz); 482 /* 483 * Insert again. mz->usage_in_excess will be updated. 484 * If excess is 0, no tree ops. 485 */ 486 __mem_cgroup_insert_exceeded(mem, mz, mctz, excess); 487 spin_unlock(&mctz->lock); 488 } 489 } 490 } 491 492 static void mem_cgroup_remove_from_trees(struct mem_cgroup *mem) 493 { 494 int node, zone; 495 struct mem_cgroup_per_zone *mz; 496 struct mem_cgroup_tree_per_zone *mctz; 497 498 for_each_node_state(node, N_POSSIBLE) { 499 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 500 mz = mem_cgroup_zoneinfo(mem, node, zone); 501 mctz = soft_limit_tree_node_zone(node, zone); 502 mem_cgroup_remove_exceeded(mem, mz, mctz); 503 } 504 } 505 } 506 507 static inline unsigned long mem_cgroup_get_excess(struct mem_cgroup *mem) 508 { 509 return res_counter_soft_limit_excess(&mem->res) >> PAGE_SHIFT; 510 } 511 512 static struct mem_cgroup_per_zone * 513 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz) 514 { 515 struct rb_node *rightmost = NULL; 516 struct mem_cgroup_per_zone *mz; 517 518 retry: 519 mz = NULL; 520 rightmost = rb_last(&mctz->rb_root); 521 if (!rightmost) 522 goto done; /* Nothing to reclaim from */ 523 524 mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node); 525 /* 526 * Remove the node now but someone else can add it back, 527 * we will to add it back at the end of reclaim to its correct 528 * position in the tree. 529 */ 530 __mem_cgroup_remove_exceeded(mz->mem, mz, mctz); 531 if (!res_counter_soft_limit_excess(&mz->mem->res) || 532 !css_tryget(&mz->mem->css)) 533 goto retry; 534 done: 535 return mz; 536 } 537 538 static struct mem_cgroup_per_zone * 539 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz) 540 { 541 struct mem_cgroup_per_zone *mz; 542 543 spin_lock(&mctz->lock); 544 mz = __mem_cgroup_largest_soft_limit_node(mctz); 545 spin_unlock(&mctz->lock); 546 return mz; 547 } 548 549 /* 550 * Implementation Note: reading percpu statistics for memcg. 551 * 552 * Both of vmstat[] and percpu_counter has threshold and do periodic 553 * synchronization to implement "quick" read. There are trade-off between 554 * reading cost and precision of value. Then, we may have a chance to implement 555 * a periodic synchronizion of counter in memcg's counter. 556 * 557 * But this _read() function is used for user interface now. The user accounts 558 * memory usage by memory cgroup and he _always_ requires exact value because 559 * he accounts memory. Even if we provide quick-and-fuzzy read, we always 560 * have to visit all online cpus and make sum. So, for now, unnecessary 561 * synchronization is not implemented. (just implemented for cpu hotplug) 562 * 563 * If there are kernel internal actions which can make use of some not-exact 564 * value, and reading all cpu value can be performance bottleneck in some 565 * common workload, threashold and synchonization as vmstat[] should be 566 * implemented. 567 */ 568 static s64 mem_cgroup_read_stat(struct mem_cgroup *mem, 569 enum mem_cgroup_stat_index idx) 570 { 571 int cpu; 572 s64 val = 0; 573 574 get_online_cpus(); 575 for_each_online_cpu(cpu) 576 val += per_cpu(mem->stat->count[idx], cpu); 577 #ifdef CONFIG_HOTPLUG_CPU 578 spin_lock(&mem->pcp_counter_lock); 579 val += mem->nocpu_base.count[idx]; 580 spin_unlock(&mem->pcp_counter_lock); 581 #endif 582 put_online_cpus(); 583 return val; 584 } 585 586 static s64 mem_cgroup_local_usage(struct mem_cgroup *mem) 587 { 588 s64 ret; 589 590 ret = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS); 591 ret += mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE); 592 return ret; 593 } 594 595 static void mem_cgroup_swap_statistics(struct mem_cgroup *mem, 596 bool charge) 597 { 598 int val = (charge) ? 1 : -1; 599 this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_SWAPOUT], val); 600 } 601 602 static void mem_cgroup_charge_statistics(struct mem_cgroup *mem, 603 struct page_cgroup *pc, 604 bool charge) 605 { 606 int val = (charge) ? 1 : -1; 607 608 preempt_disable(); 609 610 if (PageCgroupCache(pc)) 611 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_CACHE], val); 612 else 613 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_RSS], val); 614 615 if (charge) 616 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGIN_COUNT]); 617 else 618 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGOUT_COUNT]); 619 __this_cpu_inc(mem->stat->count[MEM_CGROUP_EVENTS]); 620 621 preempt_enable(); 622 } 623 624 static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem, 625 enum lru_list idx) 626 { 627 int nid, zid; 628 struct mem_cgroup_per_zone *mz; 629 u64 total = 0; 630 631 for_each_online_node(nid) 632 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 633 mz = mem_cgroup_zoneinfo(mem, nid, zid); 634 total += MEM_CGROUP_ZSTAT(mz, idx); 635 } 636 return total; 637 } 638 639 static bool __memcg_event_check(struct mem_cgroup *mem, int event_mask_shift) 640 { 641 s64 val; 642 643 val = this_cpu_read(mem->stat->count[MEM_CGROUP_EVENTS]); 644 645 return !(val & ((1 << event_mask_shift) - 1)); 646 } 647 648 /* 649 * Check events in order. 650 * 651 */ 652 static void memcg_check_events(struct mem_cgroup *mem, struct page *page) 653 { 654 /* threshold event is triggered in finer grain than soft limit */ 655 if (unlikely(__memcg_event_check(mem, THRESHOLDS_EVENTS_THRESH))) { 656 mem_cgroup_threshold(mem); 657 if (unlikely(__memcg_event_check(mem, SOFTLIMIT_EVENTS_THRESH))) 658 mem_cgroup_update_tree(mem, page); 659 } 660 } 661 662 static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont) 663 { 664 return container_of(cgroup_subsys_state(cont, 665 mem_cgroup_subsys_id), struct mem_cgroup, 666 css); 667 } 668 669 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) 670 { 671 /* 672 * mm_update_next_owner() may clear mm->owner to NULL 673 * if it races with swapoff, page migration, etc. 674 * So this can be called with p == NULL. 675 */ 676 if (unlikely(!p)) 677 return NULL; 678 679 return container_of(task_subsys_state(p, mem_cgroup_subsys_id), 680 struct mem_cgroup, css); 681 } 682 683 static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm) 684 { 685 struct mem_cgroup *mem = NULL; 686 687 if (!mm) 688 return NULL; 689 /* 690 * Because we have no locks, mm->owner's may be being moved to other 691 * cgroup. We use css_tryget() here even if this looks 692 * pessimistic (rather than adding locks here). 693 */ 694 rcu_read_lock(); 695 do { 696 mem = mem_cgroup_from_task(rcu_dereference(mm->owner)); 697 if (unlikely(!mem)) 698 break; 699 } while (!css_tryget(&mem->css)); 700 rcu_read_unlock(); 701 return mem; 702 } 703 704 /* The caller has to guarantee "mem" exists before calling this */ 705 static struct mem_cgroup *mem_cgroup_start_loop(struct mem_cgroup *mem) 706 { 707 struct cgroup_subsys_state *css; 708 int found; 709 710 if (!mem) /* ROOT cgroup has the smallest ID */ 711 return root_mem_cgroup; /*css_put/get against root is ignored*/ 712 if (!mem->use_hierarchy) { 713 if (css_tryget(&mem->css)) 714 return mem; 715 return NULL; 716 } 717 rcu_read_lock(); 718 /* 719 * searching a memory cgroup which has the smallest ID under given 720 * ROOT cgroup. (ID >= 1) 721 */ 722 css = css_get_next(&mem_cgroup_subsys, 1, &mem->css, &found); 723 if (css && css_tryget(css)) 724 mem = container_of(css, struct mem_cgroup, css); 725 else 726 mem = NULL; 727 rcu_read_unlock(); 728 return mem; 729 } 730 731 static struct mem_cgroup *mem_cgroup_get_next(struct mem_cgroup *iter, 732 struct mem_cgroup *root, 733 bool cond) 734 { 735 int nextid = css_id(&iter->css) + 1; 736 int found; 737 int hierarchy_used; 738 struct cgroup_subsys_state *css; 739 740 hierarchy_used = iter->use_hierarchy; 741 742 css_put(&iter->css); 743 /* If no ROOT, walk all, ignore hierarchy */ 744 if (!cond || (root && !hierarchy_used)) 745 return NULL; 746 747 if (!root) 748 root = root_mem_cgroup; 749 750 do { 751 iter = NULL; 752 rcu_read_lock(); 753 754 css = css_get_next(&mem_cgroup_subsys, nextid, 755 &root->css, &found); 756 if (css && css_tryget(css)) 757 iter = container_of(css, struct mem_cgroup, css); 758 rcu_read_unlock(); 759 /* If css is NULL, no more cgroups will be found */ 760 nextid = found + 1; 761 } while (css && !iter); 762 763 return iter; 764 } 765 /* 766 * for_eacn_mem_cgroup_tree() for visiting all cgroup under tree. Please 767 * be careful that "break" loop is not allowed. We have reference count. 768 * Instead of that modify "cond" to be false and "continue" to exit the loop. 769 */ 770 #define for_each_mem_cgroup_tree_cond(iter, root, cond) \ 771 for (iter = mem_cgroup_start_loop(root);\ 772 iter != NULL;\ 773 iter = mem_cgroup_get_next(iter, root, cond)) 774 775 #define for_each_mem_cgroup_tree(iter, root) \ 776 for_each_mem_cgroup_tree_cond(iter, root, true) 777 778 #define for_each_mem_cgroup_all(iter) \ 779 for_each_mem_cgroup_tree_cond(iter, NULL, true) 780 781 782 static inline bool mem_cgroup_is_root(struct mem_cgroup *mem) 783 { 784 return (mem == root_mem_cgroup); 785 } 786 787 /* 788 * Following LRU functions are allowed to be used without PCG_LOCK. 789 * Operations are called by routine of global LRU independently from memcg. 790 * What we have to take care of here is validness of pc->mem_cgroup. 791 * 792 * Changes to pc->mem_cgroup happens when 793 * 1. charge 794 * 2. moving account 795 * In typical case, "charge" is done before add-to-lru. Exception is SwapCache. 796 * It is added to LRU before charge. 797 * If PCG_USED bit is not set, page_cgroup is not added to this private LRU. 798 * When moving account, the page is not on LRU. It's isolated. 799 */ 800 801 void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru) 802 { 803 struct page_cgroup *pc; 804 struct mem_cgroup_per_zone *mz; 805 806 if (mem_cgroup_disabled()) 807 return; 808 pc = lookup_page_cgroup(page); 809 /* can happen while we handle swapcache. */ 810 if (!TestClearPageCgroupAcctLRU(pc)) 811 return; 812 VM_BUG_ON(!pc->mem_cgroup); 813 /* 814 * We don't check PCG_USED bit. It's cleared when the "page" is finally 815 * removed from global LRU. 816 */ 817 mz = page_cgroup_zoneinfo(pc); 818 MEM_CGROUP_ZSTAT(mz, lru) -= 1; 819 if (mem_cgroup_is_root(pc->mem_cgroup)) 820 return; 821 VM_BUG_ON(list_empty(&pc->lru)); 822 list_del_init(&pc->lru); 823 } 824 825 void mem_cgroup_del_lru(struct page *page) 826 { 827 mem_cgroup_del_lru_list(page, page_lru(page)); 828 } 829 830 void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru) 831 { 832 struct mem_cgroup_per_zone *mz; 833 struct page_cgroup *pc; 834 835 if (mem_cgroup_disabled()) 836 return; 837 838 pc = lookup_page_cgroup(page); 839 /* 840 * Used bit is set without atomic ops but after smp_wmb(). 841 * For making pc->mem_cgroup visible, insert smp_rmb() here. 842 */ 843 smp_rmb(); 844 /* unused or root page is not rotated. */ 845 if (!PageCgroupUsed(pc) || mem_cgroup_is_root(pc->mem_cgroup)) 846 return; 847 mz = page_cgroup_zoneinfo(pc); 848 list_move(&pc->lru, &mz->lists[lru]); 849 } 850 851 void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru) 852 { 853 struct page_cgroup *pc; 854 struct mem_cgroup_per_zone *mz; 855 856 if (mem_cgroup_disabled()) 857 return; 858 pc = lookup_page_cgroup(page); 859 VM_BUG_ON(PageCgroupAcctLRU(pc)); 860 /* 861 * Used bit is set without atomic ops but after smp_wmb(). 862 * For making pc->mem_cgroup visible, insert smp_rmb() here. 863 */ 864 smp_rmb(); 865 if (!PageCgroupUsed(pc)) 866 return; 867 868 mz = page_cgroup_zoneinfo(pc); 869 MEM_CGROUP_ZSTAT(mz, lru) += 1; 870 SetPageCgroupAcctLRU(pc); 871 if (mem_cgroup_is_root(pc->mem_cgroup)) 872 return; 873 list_add(&pc->lru, &mz->lists[lru]); 874 } 875 876 /* 877 * At handling SwapCache, pc->mem_cgroup may be changed while it's linked to 878 * lru because the page may.be reused after it's fully uncharged (because of 879 * SwapCache behavior).To handle that, unlink page_cgroup from LRU when charge 880 * it again. This function is only used to charge SwapCache. It's done under 881 * lock_page and expected that zone->lru_lock is never held. 882 */ 883 static void mem_cgroup_lru_del_before_commit_swapcache(struct page *page) 884 { 885 unsigned long flags; 886 struct zone *zone = page_zone(page); 887 struct page_cgroup *pc = lookup_page_cgroup(page); 888 889 spin_lock_irqsave(&zone->lru_lock, flags); 890 /* 891 * Forget old LRU when this page_cgroup is *not* used. This Used bit 892 * is guarded by lock_page() because the page is SwapCache. 893 */ 894 if (!PageCgroupUsed(pc)) 895 mem_cgroup_del_lru_list(page, page_lru(page)); 896 spin_unlock_irqrestore(&zone->lru_lock, flags); 897 } 898 899 static void mem_cgroup_lru_add_after_commit_swapcache(struct page *page) 900 { 901 unsigned long flags; 902 struct zone *zone = page_zone(page); 903 struct page_cgroup *pc = lookup_page_cgroup(page); 904 905 spin_lock_irqsave(&zone->lru_lock, flags); 906 /* link when the page is linked to LRU but page_cgroup isn't */ 907 if (PageLRU(page) && !PageCgroupAcctLRU(pc)) 908 mem_cgroup_add_lru_list(page, page_lru(page)); 909 spin_unlock_irqrestore(&zone->lru_lock, flags); 910 } 911 912 913 void mem_cgroup_move_lists(struct page *page, 914 enum lru_list from, enum lru_list to) 915 { 916 if (mem_cgroup_disabled()) 917 return; 918 mem_cgroup_del_lru_list(page, from); 919 mem_cgroup_add_lru_list(page, to); 920 } 921 922 int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem) 923 { 924 int ret; 925 struct mem_cgroup *curr = NULL; 926 struct task_struct *p; 927 928 p = find_lock_task_mm(task); 929 if (!p) 930 return 0; 931 curr = try_get_mem_cgroup_from_mm(p->mm); 932 task_unlock(p); 933 if (!curr) 934 return 0; 935 /* 936 * We should check use_hierarchy of "mem" not "curr". Because checking 937 * use_hierarchy of "curr" here make this function true if hierarchy is 938 * enabled in "curr" and "curr" is a child of "mem" in *cgroup* 939 * hierarchy(even if use_hierarchy is disabled in "mem"). 940 */ 941 if (mem->use_hierarchy) 942 ret = css_is_ancestor(&curr->css, &mem->css); 943 else 944 ret = (curr == mem); 945 css_put(&curr->css); 946 return ret; 947 } 948 949 static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_pages) 950 { 951 unsigned long active; 952 unsigned long inactive; 953 unsigned long gb; 954 unsigned long inactive_ratio; 955 956 inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON); 957 active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON); 958 959 gb = (inactive + active) >> (30 - PAGE_SHIFT); 960 if (gb) 961 inactive_ratio = int_sqrt(10 * gb); 962 else 963 inactive_ratio = 1; 964 965 if (present_pages) { 966 present_pages[0] = inactive; 967 present_pages[1] = active; 968 } 969 970 return inactive_ratio; 971 } 972 973 int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg) 974 { 975 unsigned long active; 976 unsigned long inactive; 977 unsigned long present_pages[2]; 978 unsigned long inactive_ratio; 979 980 inactive_ratio = calc_inactive_ratio(memcg, present_pages); 981 982 inactive = present_pages[0]; 983 active = present_pages[1]; 984 985 if (inactive * inactive_ratio < active) 986 return 1; 987 988 return 0; 989 } 990 991 int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg) 992 { 993 unsigned long active; 994 unsigned long inactive; 995 996 inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE); 997 active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE); 998 999 return (active > inactive); 1000 } 1001 1002 unsigned long mem_cgroup_zone_nr_pages(struct mem_cgroup *memcg, 1003 struct zone *zone, 1004 enum lru_list lru) 1005 { 1006 int nid = zone_to_nid(zone); 1007 int zid = zone_idx(zone); 1008 struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid); 1009 1010 return MEM_CGROUP_ZSTAT(mz, lru); 1011 } 1012 1013 struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg, 1014 struct zone *zone) 1015 { 1016 int nid = zone_to_nid(zone); 1017 int zid = zone_idx(zone); 1018 struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid); 1019 1020 return &mz->reclaim_stat; 1021 } 1022 1023 struct zone_reclaim_stat * 1024 mem_cgroup_get_reclaim_stat_from_page(struct page *page) 1025 { 1026 struct page_cgroup *pc; 1027 struct mem_cgroup_per_zone *mz; 1028 1029 if (mem_cgroup_disabled()) 1030 return NULL; 1031 1032 pc = lookup_page_cgroup(page); 1033 /* 1034 * Used bit is set without atomic ops but after smp_wmb(). 1035 * For making pc->mem_cgroup visible, insert smp_rmb() here. 1036 */ 1037 smp_rmb(); 1038 if (!PageCgroupUsed(pc)) 1039 return NULL; 1040 1041 mz = page_cgroup_zoneinfo(pc); 1042 if (!mz) 1043 return NULL; 1044 1045 return &mz->reclaim_stat; 1046 } 1047 1048 unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan, 1049 struct list_head *dst, 1050 unsigned long *scanned, int order, 1051 int mode, struct zone *z, 1052 struct mem_cgroup *mem_cont, 1053 int active, int file) 1054 { 1055 unsigned long nr_taken = 0; 1056 struct page *page; 1057 unsigned long scan; 1058 LIST_HEAD(pc_list); 1059 struct list_head *src; 1060 struct page_cgroup *pc, *tmp; 1061 int nid = zone_to_nid(z); 1062 int zid = zone_idx(z); 1063 struct mem_cgroup_per_zone *mz; 1064 int lru = LRU_FILE * file + active; 1065 int ret; 1066 1067 BUG_ON(!mem_cont); 1068 mz = mem_cgroup_zoneinfo(mem_cont, nid, zid); 1069 src = &mz->lists[lru]; 1070 1071 scan = 0; 1072 list_for_each_entry_safe_reverse(pc, tmp, src, lru) { 1073 if (scan >= nr_to_scan) 1074 break; 1075 1076 page = pc->page; 1077 if (unlikely(!PageCgroupUsed(pc))) 1078 continue; 1079 if (unlikely(!PageLRU(page))) 1080 continue; 1081 1082 scan++; 1083 ret = __isolate_lru_page(page, mode, file); 1084 switch (ret) { 1085 case 0: 1086 list_move(&page->lru, dst); 1087 mem_cgroup_del_lru(page); 1088 nr_taken += hpage_nr_pages(page); 1089 break; 1090 case -EBUSY: 1091 /* we don't affect global LRU but rotate in our LRU */ 1092 mem_cgroup_rotate_lru_list(page, page_lru(page)); 1093 break; 1094 default: 1095 break; 1096 } 1097 } 1098 1099 *scanned = scan; 1100 1101 trace_mm_vmscan_memcg_isolate(0, nr_to_scan, scan, nr_taken, 1102 0, 0, 0, mode); 1103 1104 return nr_taken; 1105 } 1106 1107 #define mem_cgroup_from_res_counter(counter, member) \ 1108 container_of(counter, struct mem_cgroup, member) 1109 1110 static bool mem_cgroup_check_under_limit(struct mem_cgroup *mem) 1111 { 1112 if (do_swap_account) { 1113 if (res_counter_check_under_limit(&mem->res) && 1114 res_counter_check_under_limit(&mem->memsw)) 1115 return true; 1116 } else 1117 if (res_counter_check_under_limit(&mem->res)) 1118 return true; 1119 return false; 1120 } 1121 1122 static unsigned int get_swappiness(struct mem_cgroup *memcg) 1123 { 1124 struct cgroup *cgrp = memcg->css.cgroup; 1125 unsigned int swappiness; 1126 1127 /* root ? */ 1128 if (cgrp->parent == NULL) 1129 return vm_swappiness; 1130 1131 spin_lock(&memcg->reclaim_param_lock); 1132 swappiness = memcg->swappiness; 1133 spin_unlock(&memcg->reclaim_param_lock); 1134 1135 return swappiness; 1136 } 1137 1138 static void mem_cgroup_start_move(struct mem_cgroup *mem) 1139 { 1140 int cpu; 1141 1142 get_online_cpus(); 1143 spin_lock(&mem->pcp_counter_lock); 1144 for_each_online_cpu(cpu) 1145 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) += 1; 1146 mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] += 1; 1147 spin_unlock(&mem->pcp_counter_lock); 1148 put_online_cpus(); 1149 1150 synchronize_rcu(); 1151 } 1152 1153 static void mem_cgroup_end_move(struct mem_cgroup *mem) 1154 { 1155 int cpu; 1156 1157 if (!mem) 1158 return; 1159 get_online_cpus(); 1160 spin_lock(&mem->pcp_counter_lock); 1161 for_each_online_cpu(cpu) 1162 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) -= 1; 1163 mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] -= 1; 1164 spin_unlock(&mem->pcp_counter_lock); 1165 put_online_cpus(); 1166 } 1167 /* 1168 * 2 routines for checking "mem" is under move_account() or not. 1169 * 1170 * mem_cgroup_stealed() - checking a cgroup is mc.from or not. This is used 1171 * for avoiding race in accounting. If true, 1172 * pc->mem_cgroup may be overwritten. 1173 * 1174 * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or 1175 * under hierarchy of moving cgroups. This is for 1176 * waiting at hith-memory prressure caused by "move". 1177 */ 1178 1179 static bool mem_cgroup_stealed(struct mem_cgroup *mem) 1180 { 1181 VM_BUG_ON(!rcu_read_lock_held()); 1182 return this_cpu_read(mem->stat->count[MEM_CGROUP_ON_MOVE]) > 0; 1183 } 1184 1185 static bool mem_cgroup_under_move(struct mem_cgroup *mem) 1186 { 1187 struct mem_cgroup *from; 1188 struct mem_cgroup *to; 1189 bool ret = false; 1190 /* 1191 * Unlike task_move routines, we access mc.to, mc.from not under 1192 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead. 1193 */ 1194 spin_lock(&mc.lock); 1195 from = mc.from; 1196 to = mc.to; 1197 if (!from) 1198 goto unlock; 1199 if (from == mem || to == mem 1200 || (mem->use_hierarchy && css_is_ancestor(&from->css, &mem->css)) 1201 || (mem->use_hierarchy && css_is_ancestor(&to->css, &mem->css))) 1202 ret = true; 1203 unlock: 1204 spin_unlock(&mc.lock); 1205 return ret; 1206 } 1207 1208 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *mem) 1209 { 1210 if (mc.moving_task && current != mc.moving_task) { 1211 if (mem_cgroup_under_move(mem)) { 1212 DEFINE_WAIT(wait); 1213 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE); 1214 /* moving charge context might have finished. */ 1215 if (mc.moving_task) 1216 schedule(); 1217 finish_wait(&mc.waitq, &wait); 1218 return true; 1219 } 1220 } 1221 return false; 1222 } 1223 1224 /** 1225 * mem_cgroup_print_oom_info: Called from OOM with tasklist_lock held in read mode. 1226 * @memcg: The memory cgroup that went over limit 1227 * @p: Task that is going to be killed 1228 * 1229 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is 1230 * enabled 1231 */ 1232 void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p) 1233 { 1234 struct cgroup *task_cgrp; 1235 struct cgroup *mem_cgrp; 1236 /* 1237 * Need a buffer in BSS, can't rely on allocations. The code relies 1238 * on the assumption that OOM is serialized for memory controller. 1239 * If this assumption is broken, revisit this code. 1240 */ 1241 static char memcg_name[PATH_MAX]; 1242 int ret; 1243 1244 if (!memcg || !p) 1245 return; 1246 1247 1248 rcu_read_lock(); 1249 1250 mem_cgrp = memcg->css.cgroup; 1251 task_cgrp = task_cgroup(p, mem_cgroup_subsys_id); 1252 1253 ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX); 1254 if (ret < 0) { 1255 /* 1256 * Unfortunately, we are unable to convert to a useful name 1257 * But we'll still print out the usage information 1258 */ 1259 rcu_read_unlock(); 1260 goto done; 1261 } 1262 rcu_read_unlock(); 1263 1264 printk(KERN_INFO "Task in %s killed", memcg_name); 1265 1266 rcu_read_lock(); 1267 ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX); 1268 if (ret < 0) { 1269 rcu_read_unlock(); 1270 goto done; 1271 } 1272 rcu_read_unlock(); 1273 1274 /* 1275 * Continues from above, so we don't need an KERN_ level 1276 */ 1277 printk(KERN_CONT " as a result of limit of %s\n", memcg_name); 1278 done: 1279 1280 printk(KERN_INFO "memory: usage %llukB, limit %llukB, failcnt %llu\n", 1281 res_counter_read_u64(&memcg->res, RES_USAGE) >> 10, 1282 res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10, 1283 res_counter_read_u64(&memcg->res, RES_FAILCNT)); 1284 printk(KERN_INFO "memory+swap: usage %llukB, limit %llukB, " 1285 "failcnt %llu\n", 1286 res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10, 1287 res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10, 1288 res_counter_read_u64(&memcg->memsw, RES_FAILCNT)); 1289 } 1290 1291 /* 1292 * This function returns the number of memcg under hierarchy tree. Returns 1293 * 1(self count) if no children. 1294 */ 1295 static int mem_cgroup_count_children(struct mem_cgroup *mem) 1296 { 1297 int num = 0; 1298 struct mem_cgroup *iter; 1299 1300 for_each_mem_cgroup_tree(iter, mem) 1301 num++; 1302 return num; 1303 } 1304 1305 /* 1306 * Return the memory (and swap, if configured) limit for a memcg. 1307 */ 1308 u64 mem_cgroup_get_limit(struct mem_cgroup *memcg) 1309 { 1310 u64 limit; 1311 u64 memsw; 1312 1313 limit = res_counter_read_u64(&memcg->res, RES_LIMIT); 1314 limit += total_swap_pages << PAGE_SHIFT; 1315 1316 memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 1317 /* 1318 * If memsw is finite and limits the amount of swap space available 1319 * to this memcg, return that limit. 1320 */ 1321 return min(limit, memsw); 1322 } 1323 1324 /* 1325 * Visit the first child (need not be the first child as per the ordering 1326 * of the cgroup list, since we track last_scanned_child) of @mem and use 1327 * that to reclaim free pages from. 1328 */ 1329 static struct mem_cgroup * 1330 mem_cgroup_select_victim(struct mem_cgroup *root_mem) 1331 { 1332 struct mem_cgroup *ret = NULL; 1333 struct cgroup_subsys_state *css; 1334 int nextid, found; 1335 1336 if (!root_mem->use_hierarchy) { 1337 css_get(&root_mem->css); 1338 ret = root_mem; 1339 } 1340 1341 while (!ret) { 1342 rcu_read_lock(); 1343 nextid = root_mem->last_scanned_child + 1; 1344 css = css_get_next(&mem_cgroup_subsys, nextid, &root_mem->css, 1345 &found); 1346 if (css && css_tryget(css)) 1347 ret = container_of(css, struct mem_cgroup, css); 1348 1349 rcu_read_unlock(); 1350 /* Updates scanning parameter */ 1351 spin_lock(&root_mem->reclaim_param_lock); 1352 if (!css) { 1353 /* this means start scan from ID:1 */ 1354 root_mem->last_scanned_child = 0; 1355 } else 1356 root_mem->last_scanned_child = found; 1357 spin_unlock(&root_mem->reclaim_param_lock); 1358 } 1359 1360 return ret; 1361 } 1362 1363 /* 1364 * Scan the hierarchy if needed to reclaim memory. We remember the last child 1365 * we reclaimed from, so that we don't end up penalizing one child extensively 1366 * based on its position in the children list. 1367 * 1368 * root_mem is the original ancestor that we've been reclaim from. 1369 * 1370 * We give up and return to the caller when we visit root_mem twice. 1371 * (other groups can be removed while we're walking....) 1372 * 1373 * If shrink==true, for avoiding to free too much, this returns immedieately. 1374 */ 1375 static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem, 1376 struct zone *zone, 1377 gfp_t gfp_mask, 1378 unsigned long reclaim_options) 1379 { 1380 struct mem_cgroup *victim; 1381 int ret, total = 0; 1382 int loop = 0; 1383 bool noswap = reclaim_options & MEM_CGROUP_RECLAIM_NOSWAP; 1384 bool shrink = reclaim_options & MEM_CGROUP_RECLAIM_SHRINK; 1385 bool check_soft = reclaim_options & MEM_CGROUP_RECLAIM_SOFT; 1386 unsigned long excess = mem_cgroup_get_excess(root_mem); 1387 1388 /* If memsw_is_minimum==1, swap-out is of-no-use. */ 1389 if (root_mem->memsw_is_minimum) 1390 noswap = true; 1391 1392 while (1) { 1393 victim = mem_cgroup_select_victim(root_mem); 1394 if (victim == root_mem) { 1395 loop++; 1396 if (loop >= 1) 1397 drain_all_stock_async(); 1398 if (loop >= 2) { 1399 /* 1400 * If we have not been able to reclaim 1401 * anything, it might because there are 1402 * no reclaimable pages under this hierarchy 1403 */ 1404 if (!check_soft || !total) { 1405 css_put(&victim->css); 1406 break; 1407 } 1408 /* 1409 * We want to do more targetted reclaim. 1410 * excess >> 2 is not to excessive so as to 1411 * reclaim too much, nor too less that we keep 1412 * coming back to reclaim from this cgroup 1413 */ 1414 if (total >= (excess >> 2) || 1415 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) { 1416 css_put(&victim->css); 1417 break; 1418 } 1419 } 1420 } 1421 if (!mem_cgroup_local_usage(victim)) { 1422 /* this cgroup's local usage == 0 */ 1423 css_put(&victim->css); 1424 continue; 1425 } 1426 /* we use swappiness of local cgroup */ 1427 if (check_soft) 1428 ret = mem_cgroup_shrink_node_zone(victim, gfp_mask, 1429 noswap, get_swappiness(victim), zone); 1430 else 1431 ret = try_to_free_mem_cgroup_pages(victim, gfp_mask, 1432 noswap, get_swappiness(victim)); 1433 css_put(&victim->css); 1434 /* 1435 * At shrinking usage, we can't check we should stop here or 1436 * reclaim more. It's depends on callers. last_scanned_child 1437 * will work enough for keeping fairness under tree. 1438 */ 1439 if (shrink) 1440 return ret; 1441 total += ret; 1442 if (check_soft) { 1443 if (res_counter_check_under_soft_limit(&root_mem->res)) 1444 return total; 1445 } else if (mem_cgroup_check_under_limit(root_mem)) 1446 return 1 + total; 1447 } 1448 return total; 1449 } 1450 1451 /* 1452 * Check OOM-Killer is already running under our hierarchy. 1453 * If someone is running, return false. 1454 */ 1455 static bool mem_cgroup_oom_lock(struct mem_cgroup *mem) 1456 { 1457 int x, lock_count = 0; 1458 struct mem_cgroup *iter; 1459 1460 for_each_mem_cgroup_tree(iter, mem) { 1461 x = atomic_inc_return(&iter->oom_lock); 1462 lock_count = max(x, lock_count); 1463 } 1464 1465 if (lock_count == 1) 1466 return true; 1467 return false; 1468 } 1469 1470 static int mem_cgroup_oom_unlock(struct mem_cgroup *mem) 1471 { 1472 struct mem_cgroup *iter; 1473 1474 /* 1475 * When a new child is created while the hierarchy is under oom, 1476 * mem_cgroup_oom_lock() may not be called. We have to use 1477 * atomic_add_unless() here. 1478 */ 1479 for_each_mem_cgroup_tree(iter, mem) 1480 atomic_add_unless(&iter->oom_lock, -1, 0); 1481 return 0; 1482 } 1483 1484 1485 static DEFINE_MUTEX(memcg_oom_mutex); 1486 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq); 1487 1488 struct oom_wait_info { 1489 struct mem_cgroup *mem; 1490 wait_queue_t wait; 1491 }; 1492 1493 static int memcg_oom_wake_function(wait_queue_t *wait, 1494 unsigned mode, int sync, void *arg) 1495 { 1496 struct mem_cgroup *wake_mem = (struct mem_cgroup *)arg; 1497 struct oom_wait_info *oom_wait_info; 1498 1499 oom_wait_info = container_of(wait, struct oom_wait_info, wait); 1500 1501 if (oom_wait_info->mem == wake_mem) 1502 goto wakeup; 1503 /* if no hierarchy, no match */ 1504 if (!oom_wait_info->mem->use_hierarchy || !wake_mem->use_hierarchy) 1505 return 0; 1506 /* 1507 * Both of oom_wait_info->mem and wake_mem are stable under us. 1508 * Then we can use css_is_ancestor without taking care of RCU. 1509 */ 1510 if (!css_is_ancestor(&oom_wait_info->mem->css, &wake_mem->css) && 1511 !css_is_ancestor(&wake_mem->css, &oom_wait_info->mem->css)) 1512 return 0; 1513 1514 wakeup: 1515 return autoremove_wake_function(wait, mode, sync, arg); 1516 } 1517 1518 static void memcg_wakeup_oom(struct mem_cgroup *mem) 1519 { 1520 /* for filtering, pass "mem" as argument. */ 1521 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, mem); 1522 } 1523 1524 static void memcg_oom_recover(struct mem_cgroup *mem) 1525 { 1526 if (mem && atomic_read(&mem->oom_lock)) 1527 memcg_wakeup_oom(mem); 1528 } 1529 1530 /* 1531 * try to call OOM killer. returns false if we should exit memory-reclaim loop. 1532 */ 1533 bool mem_cgroup_handle_oom(struct mem_cgroup *mem, gfp_t mask) 1534 { 1535 struct oom_wait_info owait; 1536 bool locked, need_to_kill; 1537 1538 owait.mem = mem; 1539 owait.wait.flags = 0; 1540 owait.wait.func = memcg_oom_wake_function; 1541 owait.wait.private = current; 1542 INIT_LIST_HEAD(&owait.wait.task_list); 1543 need_to_kill = true; 1544 /* At first, try to OOM lock hierarchy under mem.*/ 1545 mutex_lock(&memcg_oom_mutex); 1546 locked = mem_cgroup_oom_lock(mem); 1547 /* 1548 * Even if signal_pending(), we can't quit charge() loop without 1549 * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL 1550 * under OOM is always welcomed, use TASK_KILLABLE here. 1551 */ 1552 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE); 1553 if (!locked || mem->oom_kill_disable) 1554 need_to_kill = false; 1555 if (locked) 1556 mem_cgroup_oom_notify(mem); 1557 mutex_unlock(&memcg_oom_mutex); 1558 1559 if (need_to_kill) { 1560 finish_wait(&memcg_oom_waitq, &owait.wait); 1561 mem_cgroup_out_of_memory(mem, mask); 1562 } else { 1563 schedule(); 1564 finish_wait(&memcg_oom_waitq, &owait.wait); 1565 } 1566 mutex_lock(&memcg_oom_mutex); 1567 mem_cgroup_oom_unlock(mem); 1568 memcg_wakeup_oom(mem); 1569 mutex_unlock(&memcg_oom_mutex); 1570 1571 if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current)) 1572 return false; 1573 /* Give chance to dying process */ 1574 schedule_timeout(1); 1575 return true; 1576 } 1577 1578 /* 1579 * Currently used to update mapped file statistics, but the routine can be 1580 * generalized to update other statistics as well. 1581 * 1582 * Notes: Race condition 1583 * 1584 * We usually use page_cgroup_lock() for accessing page_cgroup member but 1585 * it tends to be costly. But considering some conditions, we doesn't need 1586 * to do so _always_. 1587 * 1588 * Considering "charge", lock_page_cgroup() is not required because all 1589 * file-stat operations happen after a page is attached to radix-tree. There 1590 * are no race with "charge". 1591 * 1592 * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup 1593 * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even 1594 * if there are race with "uncharge". Statistics itself is properly handled 1595 * by flags. 1596 * 1597 * Considering "move", this is an only case we see a race. To make the race 1598 * small, we check MEM_CGROUP_ON_MOVE percpu value and detect there are 1599 * possibility of race condition. If there is, we take a lock. 1600 */ 1601 1602 void mem_cgroup_update_page_stat(struct page *page, 1603 enum mem_cgroup_page_stat_item idx, int val) 1604 { 1605 struct mem_cgroup *mem; 1606 struct page_cgroup *pc = lookup_page_cgroup(page); 1607 bool need_unlock = false; 1608 unsigned long uninitialized_var(flags); 1609 1610 if (unlikely(!pc)) 1611 return; 1612 1613 rcu_read_lock(); 1614 mem = pc->mem_cgroup; 1615 if (unlikely(!mem || !PageCgroupUsed(pc))) 1616 goto out; 1617 /* pc->mem_cgroup is unstable ? */ 1618 if (unlikely(mem_cgroup_stealed(mem))) { 1619 /* take a lock against to access pc->mem_cgroup */ 1620 move_lock_page_cgroup(pc, &flags); 1621 need_unlock = true; 1622 mem = pc->mem_cgroup; 1623 if (!mem || !PageCgroupUsed(pc)) 1624 goto out; 1625 } 1626 1627 switch (idx) { 1628 case MEMCG_NR_FILE_MAPPED: 1629 if (val > 0) 1630 SetPageCgroupFileMapped(pc); 1631 else if (!page_mapped(page)) 1632 ClearPageCgroupFileMapped(pc); 1633 idx = MEM_CGROUP_STAT_FILE_MAPPED; 1634 break; 1635 default: 1636 BUG(); 1637 } 1638 1639 this_cpu_add(mem->stat->count[idx], val); 1640 1641 out: 1642 if (unlikely(need_unlock)) 1643 move_unlock_page_cgroup(pc, &flags); 1644 rcu_read_unlock(); 1645 return; 1646 } 1647 EXPORT_SYMBOL(mem_cgroup_update_page_stat); 1648 1649 /* 1650 * size of first charge trial. "32" comes from vmscan.c's magic value. 1651 * TODO: maybe necessary to use big numbers in big irons. 1652 */ 1653 #define CHARGE_SIZE (32 * PAGE_SIZE) 1654 struct memcg_stock_pcp { 1655 struct mem_cgroup *cached; /* this never be root cgroup */ 1656 int charge; 1657 struct work_struct work; 1658 }; 1659 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock); 1660 static atomic_t memcg_drain_count; 1661 1662 /* 1663 * Try to consume stocked charge on this cpu. If success, PAGE_SIZE is consumed 1664 * from local stock and true is returned. If the stock is 0 or charges from a 1665 * cgroup which is not current target, returns false. This stock will be 1666 * refilled. 1667 */ 1668 static bool consume_stock(struct mem_cgroup *mem) 1669 { 1670 struct memcg_stock_pcp *stock; 1671 bool ret = true; 1672 1673 stock = &get_cpu_var(memcg_stock); 1674 if (mem == stock->cached && stock->charge) 1675 stock->charge -= PAGE_SIZE; 1676 else /* need to call res_counter_charge */ 1677 ret = false; 1678 put_cpu_var(memcg_stock); 1679 return ret; 1680 } 1681 1682 /* 1683 * Returns stocks cached in percpu to res_counter and reset cached information. 1684 */ 1685 static void drain_stock(struct memcg_stock_pcp *stock) 1686 { 1687 struct mem_cgroup *old = stock->cached; 1688 1689 if (stock->charge) { 1690 res_counter_uncharge(&old->res, stock->charge); 1691 if (do_swap_account) 1692 res_counter_uncharge(&old->memsw, stock->charge); 1693 } 1694 stock->cached = NULL; 1695 stock->charge = 0; 1696 } 1697 1698 /* 1699 * This must be called under preempt disabled or must be called by 1700 * a thread which is pinned to local cpu. 1701 */ 1702 static void drain_local_stock(struct work_struct *dummy) 1703 { 1704 struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock); 1705 drain_stock(stock); 1706 } 1707 1708 /* 1709 * Cache charges(val) which is from res_counter, to local per_cpu area. 1710 * This will be consumed by consume_stock() function, later. 1711 */ 1712 static void refill_stock(struct mem_cgroup *mem, int val) 1713 { 1714 struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock); 1715 1716 if (stock->cached != mem) { /* reset if necessary */ 1717 drain_stock(stock); 1718 stock->cached = mem; 1719 } 1720 stock->charge += val; 1721 put_cpu_var(memcg_stock); 1722 } 1723 1724 /* 1725 * Tries to drain stocked charges in other cpus. This function is asynchronous 1726 * and just put a work per cpu for draining localy on each cpu. Caller can 1727 * expects some charges will be back to res_counter later but cannot wait for 1728 * it. 1729 */ 1730 static void drain_all_stock_async(void) 1731 { 1732 int cpu; 1733 /* This function is for scheduling "drain" in asynchronous way. 1734 * The result of "drain" is not directly handled by callers. Then, 1735 * if someone is calling drain, we don't have to call drain more. 1736 * Anyway, WORK_STRUCT_PENDING check in queue_work_on() will catch if 1737 * there is a race. We just do loose check here. 1738 */ 1739 if (atomic_read(&memcg_drain_count)) 1740 return; 1741 /* Notify other cpus that system-wide "drain" is running */ 1742 atomic_inc(&memcg_drain_count); 1743 get_online_cpus(); 1744 for_each_online_cpu(cpu) { 1745 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); 1746 schedule_work_on(cpu, &stock->work); 1747 } 1748 put_online_cpus(); 1749 atomic_dec(&memcg_drain_count); 1750 /* We don't wait for flush_work */ 1751 } 1752 1753 /* This is a synchronous drain interface. */ 1754 static void drain_all_stock_sync(void) 1755 { 1756 /* called when force_empty is called */ 1757 atomic_inc(&memcg_drain_count); 1758 schedule_on_each_cpu(drain_local_stock); 1759 atomic_dec(&memcg_drain_count); 1760 } 1761 1762 /* 1763 * This function drains percpu counter value from DEAD cpu and 1764 * move it to local cpu. Note that this function can be preempted. 1765 */ 1766 static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *mem, int cpu) 1767 { 1768 int i; 1769 1770 spin_lock(&mem->pcp_counter_lock); 1771 for (i = 0; i < MEM_CGROUP_STAT_DATA; i++) { 1772 s64 x = per_cpu(mem->stat->count[i], cpu); 1773 1774 per_cpu(mem->stat->count[i], cpu) = 0; 1775 mem->nocpu_base.count[i] += x; 1776 } 1777 /* need to clear ON_MOVE value, works as a kind of lock. */ 1778 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) = 0; 1779 spin_unlock(&mem->pcp_counter_lock); 1780 } 1781 1782 static void synchronize_mem_cgroup_on_move(struct mem_cgroup *mem, int cpu) 1783 { 1784 int idx = MEM_CGROUP_ON_MOVE; 1785 1786 spin_lock(&mem->pcp_counter_lock); 1787 per_cpu(mem->stat->count[idx], cpu) = mem->nocpu_base.count[idx]; 1788 spin_unlock(&mem->pcp_counter_lock); 1789 } 1790 1791 static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb, 1792 unsigned long action, 1793 void *hcpu) 1794 { 1795 int cpu = (unsigned long)hcpu; 1796 struct memcg_stock_pcp *stock; 1797 struct mem_cgroup *iter; 1798 1799 if ((action == CPU_ONLINE)) { 1800 for_each_mem_cgroup_all(iter) 1801 synchronize_mem_cgroup_on_move(iter, cpu); 1802 return NOTIFY_OK; 1803 } 1804 1805 if ((action != CPU_DEAD) || action != CPU_DEAD_FROZEN) 1806 return NOTIFY_OK; 1807 1808 for_each_mem_cgroup_all(iter) 1809 mem_cgroup_drain_pcp_counter(iter, cpu); 1810 1811 stock = &per_cpu(memcg_stock, cpu); 1812 drain_stock(stock); 1813 return NOTIFY_OK; 1814 } 1815 1816 1817 /* See __mem_cgroup_try_charge() for details */ 1818 enum { 1819 CHARGE_OK, /* success */ 1820 CHARGE_RETRY, /* need to retry but retry is not bad */ 1821 CHARGE_NOMEM, /* we can't do more. return -ENOMEM */ 1822 CHARGE_WOULDBLOCK, /* GFP_WAIT wasn't set and no enough res. */ 1823 CHARGE_OOM_DIE, /* the current is killed because of OOM */ 1824 }; 1825 1826 static int __mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask, 1827 int csize, bool oom_check) 1828 { 1829 struct mem_cgroup *mem_over_limit; 1830 struct res_counter *fail_res; 1831 unsigned long flags = 0; 1832 int ret; 1833 1834 ret = res_counter_charge(&mem->res, csize, &fail_res); 1835 1836 if (likely(!ret)) { 1837 if (!do_swap_account) 1838 return CHARGE_OK; 1839 ret = res_counter_charge(&mem->memsw, csize, &fail_res); 1840 if (likely(!ret)) 1841 return CHARGE_OK; 1842 1843 mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw); 1844 flags |= MEM_CGROUP_RECLAIM_NOSWAP; 1845 } else 1846 mem_over_limit = mem_cgroup_from_res_counter(fail_res, res); 1847 1848 if (csize > PAGE_SIZE) /* change csize and retry */ 1849 return CHARGE_RETRY; 1850 1851 if (!(gfp_mask & __GFP_WAIT)) 1852 return CHARGE_WOULDBLOCK; 1853 1854 ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL, 1855 gfp_mask, flags); 1856 /* 1857 * try_to_free_mem_cgroup_pages() might not give us a full 1858 * picture of reclaim. Some pages are reclaimed and might be 1859 * moved to swap cache or just unmapped from the cgroup. 1860 * Check the limit again to see if the reclaim reduced the 1861 * current usage of the cgroup before giving up 1862 */ 1863 if (ret || mem_cgroup_check_under_limit(mem_over_limit)) 1864 return CHARGE_RETRY; 1865 1866 /* 1867 * At task move, charge accounts can be doubly counted. So, it's 1868 * better to wait until the end of task_move if something is going on. 1869 */ 1870 if (mem_cgroup_wait_acct_move(mem_over_limit)) 1871 return CHARGE_RETRY; 1872 1873 /* If we don't need to call oom-killer at el, return immediately */ 1874 if (!oom_check) 1875 return CHARGE_NOMEM; 1876 /* check OOM */ 1877 if (!mem_cgroup_handle_oom(mem_over_limit, gfp_mask)) 1878 return CHARGE_OOM_DIE; 1879 1880 return CHARGE_RETRY; 1881 } 1882 1883 /* 1884 * Unlike exported interface, "oom" parameter is added. if oom==true, 1885 * oom-killer can be invoked. 1886 */ 1887 static int __mem_cgroup_try_charge(struct mm_struct *mm, 1888 gfp_t gfp_mask, 1889 struct mem_cgroup **memcg, bool oom, 1890 int page_size) 1891 { 1892 int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES; 1893 struct mem_cgroup *mem = NULL; 1894 int ret; 1895 int csize = max(CHARGE_SIZE, (unsigned long) page_size); 1896 1897 /* 1898 * Unlike gloval-vm's OOM-kill, we're not in memory shortage 1899 * in system level. So, allow to go ahead dying process in addition to 1900 * MEMDIE process. 1901 */ 1902 if (unlikely(test_thread_flag(TIF_MEMDIE) 1903 || fatal_signal_pending(current))) 1904 goto bypass; 1905 1906 /* 1907 * We always charge the cgroup the mm_struct belongs to. 1908 * The mm_struct's mem_cgroup changes on task migration if the 1909 * thread group leader migrates. It's possible that mm is not 1910 * set, if so charge the init_mm (happens for pagecache usage). 1911 */ 1912 if (!*memcg && !mm) 1913 goto bypass; 1914 again: 1915 if (*memcg) { /* css should be a valid one */ 1916 mem = *memcg; 1917 VM_BUG_ON(css_is_removed(&mem->css)); 1918 if (mem_cgroup_is_root(mem)) 1919 goto done; 1920 if (page_size == PAGE_SIZE && consume_stock(mem)) 1921 goto done; 1922 css_get(&mem->css); 1923 } else { 1924 struct task_struct *p; 1925 1926 rcu_read_lock(); 1927 p = rcu_dereference(mm->owner); 1928 /* 1929 * Because we don't have task_lock(), "p" can exit. 1930 * In that case, "mem" can point to root or p can be NULL with 1931 * race with swapoff. Then, we have small risk of mis-accouning. 1932 * But such kind of mis-account by race always happens because 1933 * we don't have cgroup_mutex(). It's overkill and we allo that 1934 * small race, here. 1935 * (*) swapoff at el will charge against mm-struct not against 1936 * task-struct. So, mm->owner can be NULL. 1937 */ 1938 mem = mem_cgroup_from_task(p); 1939 if (!mem || mem_cgroup_is_root(mem)) { 1940 rcu_read_unlock(); 1941 goto done; 1942 } 1943 if (page_size == PAGE_SIZE && consume_stock(mem)) { 1944 /* 1945 * It seems dagerous to access memcg without css_get(). 1946 * But considering how consume_stok works, it's not 1947 * necessary. If consume_stock success, some charges 1948 * from this memcg are cached on this cpu. So, we 1949 * don't need to call css_get()/css_tryget() before 1950 * calling consume_stock(). 1951 */ 1952 rcu_read_unlock(); 1953 goto done; 1954 } 1955 /* after here, we may be blocked. we need to get refcnt */ 1956 if (!css_tryget(&mem->css)) { 1957 rcu_read_unlock(); 1958 goto again; 1959 } 1960 rcu_read_unlock(); 1961 } 1962 1963 do { 1964 bool oom_check; 1965 1966 /* If killed, bypass charge */ 1967 if (fatal_signal_pending(current)) { 1968 css_put(&mem->css); 1969 goto bypass; 1970 } 1971 1972 oom_check = false; 1973 if (oom && !nr_oom_retries) { 1974 oom_check = true; 1975 nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES; 1976 } 1977 1978 ret = __mem_cgroup_do_charge(mem, gfp_mask, csize, oom_check); 1979 1980 switch (ret) { 1981 case CHARGE_OK: 1982 break; 1983 case CHARGE_RETRY: /* not in OOM situation but retry */ 1984 csize = page_size; 1985 css_put(&mem->css); 1986 mem = NULL; 1987 goto again; 1988 case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */ 1989 css_put(&mem->css); 1990 goto nomem; 1991 case CHARGE_NOMEM: /* OOM routine works */ 1992 if (!oom) { 1993 css_put(&mem->css); 1994 goto nomem; 1995 } 1996 /* If oom, we never return -ENOMEM */ 1997 nr_oom_retries--; 1998 break; 1999 case CHARGE_OOM_DIE: /* Killed by OOM Killer */ 2000 css_put(&mem->css); 2001 goto bypass; 2002 } 2003 } while (ret != CHARGE_OK); 2004 2005 if (csize > page_size) 2006 refill_stock(mem, csize - page_size); 2007 css_put(&mem->css); 2008 done: 2009 *memcg = mem; 2010 return 0; 2011 nomem: 2012 *memcg = NULL; 2013 return -ENOMEM; 2014 bypass: 2015 *memcg = NULL; 2016 return 0; 2017 } 2018 2019 /* 2020 * Somemtimes we have to undo a charge we got by try_charge(). 2021 * This function is for that and do uncharge, put css's refcnt. 2022 * gotten by try_charge(). 2023 */ 2024 static void __mem_cgroup_cancel_charge(struct mem_cgroup *mem, 2025 unsigned long count) 2026 { 2027 if (!mem_cgroup_is_root(mem)) { 2028 res_counter_uncharge(&mem->res, PAGE_SIZE * count); 2029 if (do_swap_account) 2030 res_counter_uncharge(&mem->memsw, PAGE_SIZE * count); 2031 } 2032 } 2033 2034 static void mem_cgroup_cancel_charge(struct mem_cgroup *mem, 2035 int page_size) 2036 { 2037 __mem_cgroup_cancel_charge(mem, page_size >> PAGE_SHIFT); 2038 } 2039 2040 /* 2041 * A helper function to get mem_cgroup from ID. must be called under 2042 * rcu_read_lock(). The caller must check css_is_removed() or some if 2043 * it's concern. (dropping refcnt from swap can be called against removed 2044 * memcg.) 2045 */ 2046 static struct mem_cgroup *mem_cgroup_lookup(unsigned short id) 2047 { 2048 struct cgroup_subsys_state *css; 2049 2050 /* ID 0 is unused ID */ 2051 if (!id) 2052 return NULL; 2053 css = css_lookup(&mem_cgroup_subsys, id); 2054 if (!css) 2055 return NULL; 2056 return container_of(css, struct mem_cgroup, css); 2057 } 2058 2059 struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page) 2060 { 2061 struct mem_cgroup *mem = NULL; 2062 struct page_cgroup *pc; 2063 unsigned short id; 2064 swp_entry_t ent; 2065 2066 VM_BUG_ON(!PageLocked(page)); 2067 2068 pc = lookup_page_cgroup(page); 2069 lock_page_cgroup(pc); 2070 if (PageCgroupUsed(pc)) { 2071 mem = pc->mem_cgroup; 2072 if (mem && !css_tryget(&mem->css)) 2073 mem = NULL; 2074 } else if (PageSwapCache(page)) { 2075 ent.val = page_private(page); 2076 id = lookup_swap_cgroup(ent); 2077 rcu_read_lock(); 2078 mem = mem_cgroup_lookup(id); 2079 if (mem && !css_tryget(&mem->css)) 2080 mem = NULL; 2081 rcu_read_unlock(); 2082 } 2083 unlock_page_cgroup(pc); 2084 return mem; 2085 } 2086 2087 /* 2088 * commit a charge got by __mem_cgroup_try_charge() and makes page_cgroup to be 2089 * USED state. If already USED, uncharge and return. 2090 */ 2091 static void ____mem_cgroup_commit_charge(struct mem_cgroup *mem, 2092 struct page_cgroup *pc, 2093 enum charge_type ctype) 2094 { 2095 pc->mem_cgroup = mem; 2096 /* 2097 * We access a page_cgroup asynchronously without lock_page_cgroup(). 2098 * Especially when a page_cgroup is taken from a page, pc->mem_cgroup 2099 * is accessed after testing USED bit. To make pc->mem_cgroup visible 2100 * before USED bit, we need memory barrier here. 2101 * See mem_cgroup_add_lru_list(), etc. 2102 */ 2103 smp_wmb(); 2104 switch (ctype) { 2105 case MEM_CGROUP_CHARGE_TYPE_CACHE: 2106 case MEM_CGROUP_CHARGE_TYPE_SHMEM: 2107 SetPageCgroupCache(pc); 2108 SetPageCgroupUsed(pc); 2109 break; 2110 case MEM_CGROUP_CHARGE_TYPE_MAPPED: 2111 ClearPageCgroupCache(pc); 2112 SetPageCgroupUsed(pc); 2113 break; 2114 default: 2115 break; 2116 } 2117 2118 mem_cgroup_charge_statistics(mem, pc, true); 2119 } 2120 2121 static void __mem_cgroup_commit_charge(struct mem_cgroup *mem, 2122 struct page_cgroup *pc, 2123 enum charge_type ctype, 2124 int page_size) 2125 { 2126 int i; 2127 int count = page_size >> PAGE_SHIFT; 2128 2129 /* try_charge() can return NULL to *memcg, taking care of it. */ 2130 if (!mem) 2131 return; 2132 2133 lock_page_cgroup(pc); 2134 if (unlikely(PageCgroupUsed(pc))) { 2135 unlock_page_cgroup(pc); 2136 mem_cgroup_cancel_charge(mem, page_size); 2137 return; 2138 } 2139 2140 /* 2141 * we don't need page_cgroup_lock about tail pages, becase they are not 2142 * accessed by any other context at this point. 2143 */ 2144 for (i = 0; i < count; i++) 2145 ____mem_cgroup_commit_charge(mem, pc + i, ctype); 2146 2147 unlock_page_cgroup(pc); 2148 /* 2149 * "charge_statistics" updated event counter. Then, check it. 2150 * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree. 2151 * if they exceeds softlimit. 2152 */ 2153 memcg_check_events(mem, pc->page); 2154 } 2155 2156 /** 2157 * __mem_cgroup_move_account - move account of the page 2158 * @pc: page_cgroup of the page. 2159 * @from: mem_cgroup which the page is moved from. 2160 * @to: mem_cgroup which the page is moved to. @from != @to. 2161 * @uncharge: whether we should call uncharge and css_put against @from. 2162 * 2163 * The caller must confirm following. 2164 * - page is not on LRU (isolate_page() is useful.) 2165 * - the pc is locked, used, and ->mem_cgroup points to @from. 2166 * 2167 * This function doesn't do "charge" nor css_get to new cgroup. It should be 2168 * done by a caller(__mem_cgroup_try_charge would be usefull). If @uncharge is 2169 * true, this function does "uncharge" from old cgroup, but it doesn't if 2170 * @uncharge is false, so a caller should do "uncharge". 2171 */ 2172 2173 static void __mem_cgroup_move_account(struct page_cgroup *pc, 2174 struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge) 2175 { 2176 VM_BUG_ON(from == to); 2177 VM_BUG_ON(PageLRU(pc->page)); 2178 VM_BUG_ON(!page_is_cgroup_locked(pc)); 2179 VM_BUG_ON(!PageCgroupUsed(pc)); 2180 VM_BUG_ON(pc->mem_cgroup != from); 2181 2182 if (PageCgroupFileMapped(pc)) { 2183 /* Update mapped_file data for mem_cgroup */ 2184 preempt_disable(); 2185 __this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]); 2186 __this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]); 2187 preempt_enable(); 2188 } 2189 mem_cgroup_charge_statistics(from, pc, false); 2190 if (uncharge) 2191 /* This is not "cancel", but cancel_charge does all we need. */ 2192 mem_cgroup_cancel_charge(from, PAGE_SIZE); 2193 2194 /* caller should have done css_get */ 2195 pc->mem_cgroup = to; 2196 mem_cgroup_charge_statistics(to, pc, true); 2197 /* 2198 * We charges against "to" which may not have any tasks. Then, "to" 2199 * can be under rmdir(). But in current implementation, caller of 2200 * this function is just force_empty() and move charge, so it's 2201 * garanteed that "to" is never removed. So, we don't check rmdir 2202 * status here. 2203 */ 2204 } 2205 2206 /* 2207 * check whether the @pc is valid for moving account and call 2208 * __mem_cgroup_move_account() 2209 */ 2210 static int mem_cgroup_move_account(struct page_cgroup *pc, 2211 struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge) 2212 { 2213 int ret = -EINVAL; 2214 unsigned long flags; 2215 2216 lock_page_cgroup(pc); 2217 if (PageCgroupUsed(pc) && pc->mem_cgroup == from) { 2218 move_lock_page_cgroup(pc, &flags); 2219 __mem_cgroup_move_account(pc, from, to, uncharge); 2220 move_unlock_page_cgroup(pc, &flags); 2221 ret = 0; 2222 } 2223 unlock_page_cgroup(pc); 2224 /* 2225 * check events 2226 */ 2227 memcg_check_events(to, pc->page); 2228 memcg_check_events(from, pc->page); 2229 return ret; 2230 } 2231 2232 /* 2233 * move charges to its parent. 2234 */ 2235 2236 static int mem_cgroup_move_parent(struct page_cgroup *pc, 2237 struct mem_cgroup *child, 2238 gfp_t gfp_mask) 2239 { 2240 struct page *page = pc->page; 2241 struct cgroup *cg = child->css.cgroup; 2242 struct cgroup *pcg = cg->parent; 2243 struct mem_cgroup *parent; 2244 int ret; 2245 2246 /* Is ROOT ? */ 2247 if (!pcg) 2248 return -EINVAL; 2249 2250 ret = -EBUSY; 2251 if (!get_page_unless_zero(page)) 2252 goto out; 2253 if (isolate_lru_page(page)) 2254 goto put; 2255 2256 parent = mem_cgroup_from_cont(pcg); 2257 ret = __mem_cgroup_try_charge(NULL, gfp_mask, &parent, false, 2258 PAGE_SIZE); 2259 if (ret || !parent) 2260 goto put_back; 2261 2262 ret = mem_cgroup_move_account(pc, child, parent, true); 2263 if (ret) 2264 mem_cgroup_cancel_charge(parent, PAGE_SIZE); 2265 put_back: 2266 putback_lru_page(page); 2267 put: 2268 put_page(page); 2269 out: 2270 return ret; 2271 } 2272 2273 /* 2274 * Charge the memory controller for page usage. 2275 * Return 2276 * 0 if the charge was successful 2277 * < 0 if the cgroup is over its limit 2278 */ 2279 static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm, 2280 gfp_t gfp_mask, enum charge_type ctype) 2281 { 2282 struct mem_cgroup *mem = NULL; 2283 struct page_cgroup *pc; 2284 int ret; 2285 int page_size = PAGE_SIZE; 2286 2287 if (PageTransHuge(page)) { 2288 page_size <<= compound_order(page); 2289 VM_BUG_ON(!PageTransHuge(page)); 2290 } 2291 2292 pc = lookup_page_cgroup(page); 2293 /* can happen at boot */ 2294 if (unlikely(!pc)) 2295 return 0; 2296 prefetchw(pc); 2297 2298 ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, true, page_size); 2299 if (ret || !mem) 2300 return ret; 2301 2302 __mem_cgroup_commit_charge(mem, pc, ctype, page_size); 2303 return 0; 2304 } 2305 2306 int mem_cgroup_newpage_charge(struct page *page, 2307 struct mm_struct *mm, gfp_t gfp_mask) 2308 { 2309 if (mem_cgroup_disabled()) 2310 return 0; 2311 /* 2312 * If already mapped, we don't have to account. 2313 * If page cache, page->mapping has address_space. 2314 * But page->mapping may have out-of-use anon_vma pointer, 2315 * detecit it by PageAnon() check. newly-mapped-anon's page->mapping 2316 * is NULL. 2317 */ 2318 if (page_mapped(page) || (page->mapping && !PageAnon(page))) 2319 return 0; 2320 if (unlikely(!mm)) 2321 mm = &init_mm; 2322 return mem_cgroup_charge_common(page, mm, gfp_mask, 2323 MEM_CGROUP_CHARGE_TYPE_MAPPED); 2324 } 2325 2326 static void 2327 __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr, 2328 enum charge_type ctype); 2329 2330 int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm, 2331 gfp_t gfp_mask) 2332 { 2333 int ret; 2334 2335 if (mem_cgroup_disabled()) 2336 return 0; 2337 if (PageCompound(page)) 2338 return 0; 2339 /* 2340 * Corner case handling. This is called from add_to_page_cache() 2341 * in usual. But some FS (shmem) precharges this page before calling it 2342 * and call add_to_page_cache() with GFP_NOWAIT. 2343 * 2344 * For GFP_NOWAIT case, the page may be pre-charged before calling 2345 * add_to_page_cache(). (See shmem.c) check it here and avoid to call 2346 * charge twice. (It works but has to pay a bit larger cost.) 2347 * And when the page is SwapCache, it should take swap information 2348 * into account. This is under lock_page() now. 2349 */ 2350 if (!(gfp_mask & __GFP_WAIT)) { 2351 struct page_cgroup *pc; 2352 2353 pc = lookup_page_cgroup(page); 2354 if (!pc) 2355 return 0; 2356 lock_page_cgroup(pc); 2357 if (PageCgroupUsed(pc)) { 2358 unlock_page_cgroup(pc); 2359 return 0; 2360 } 2361 unlock_page_cgroup(pc); 2362 } 2363 2364 if (unlikely(!mm)) 2365 mm = &init_mm; 2366 2367 if (page_is_file_cache(page)) 2368 return mem_cgroup_charge_common(page, mm, gfp_mask, 2369 MEM_CGROUP_CHARGE_TYPE_CACHE); 2370 2371 /* shmem */ 2372 if (PageSwapCache(page)) { 2373 struct mem_cgroup *mem = NULL; 2374 2375 ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem); 2376 if (!ret) 2377 __mem_cgroup_commit_charge_swapin(page, mem, 2378 MEM_CGROUP_CHARGE_TYPE_SHMEM); 2379 } else 2380 ret = mem_cgroup_charge_common(page, mm, gfp_mask, 2381 MEM_CGROUP_CHARGE_TYPE_SHMEM); 2382 2383 return ret; 2384 } 2385 2386 /* 2387 * While swap-in, try_charge -> commit or cancel, the page is locked. 2388 * And when try_charge() successfully returns, one refcnt to memcg without 2389 * struct page_cgroup is acquired. This refcnt will be consumed by 2390 * "commit()" or removed by "cancel()" 2391 */ 2392 int mem_cgroup_try_charge_swapin(struct mm_struct *mm, 2393 struct page *page, 2394 gfp_t mask, struct mem_cgroup **ptr) 2395 { 2396 struct mem_cgroup *mem; 2397 int ret; 2398 2399 if (mem_cgroup_disabled()) 2400 return 0; 2401 2402 if (!do_swap_account) 2403 goto charge_cur_mm; 2404 /* 2405 * A racing thread's fault, or swapoff, may have already updated 2406 * the pte, and even removed page from swap cache: in those cases 2407 * do_swap_page()'s pte_same() test will fail; but there's also a 2408 * KSM case which does need to charge the page. 2409 */ 2410 if (!PageSwapCache(page)) 2411 goto charge_cur_mm; 2412 mem = try_get_mem_cgroup_from_page(page); 2413 if (!mem) 2414 goto charge_cur_mm; 2415 *ptr = mem; 2416 ret = __mem_cgroup_try_charge(NULL, mask, ptr, true, PAGE_SIZE); 2417 css_put(&mem->css); 2418 return ret; 2419 charge_cur_mm: 2420 if (unlikely(!mm)) 2421 mm = &init_mm; 2422 return __mem_cgroup_try_charge(mm, mask, ptr, true, PAGE_SIZE); 2423 } 2424 2425 static void 2426 __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr, 2427 enum charge_type ctype) 2428 { 2429 struct page_cgroup *pc; 2430 2431 if (mem_cgroup_disabled()) 2432 return; 2433 if (!ptr) 2434 return; 2435 cgroup_exclude_rmdir(&ptr->css); 2436 pc = lookup_page_cgroup(page); 2437 mem_cgroup_lru_del_before_commit_swapcache(page); 2438 __mem_cgroup_commit_charge(ptr, pc, ctype, PAGE_SIZE); 2439 mem_cgroup_lru_add_after_commit_swapcache(page); 2440 /* 2441 * Now swap is on-memory. This means this page may be 2442 * counted both as mem and swap....double count. 2443 * Fix it by uncharging from memsw. Basically, this SwapCache is stable 2444 * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page() 2445 * may call delete_from_swap_cache() before reach here. 2446 */ 2447 if (do_swap_account && PageSwapCache(page)) { 2448 swp_entry_t ent = {.val = page_private(page)}; 2449 unsigned short id; 2450 struct mem_cgroup *memcg; 2451 2452 id = swap_cgroup_record(ent, 0); 2453 rcu_read_lock(); 2454 memcg = mem_cgroup_lookup(id); 2455 if (memcg) { 2456 /* 2457 * This recorded memcg can be obsolete one. So, avoid 2458 * calling css_tryget 2459 */ 2460 if (!mem_cgroup_is_root(memcg)) 2461 res_counter_uncharge(&memcg->memsw, PAGE_SIZE); 2462 mem_cgroup_swap_statistics(memcg, false); 2463 mem_cgroup_put(memcg); 2464 } 2465 rcu_read_unlock(); 2466 } 2467 /* 2468 * At swapin, we may charge account against cgroup which has no tasks. 2469 * So, rmdir()->pre_destroy() can be called while we do this charge. 2470 * In that case, we need to call pre_destroy() again. check it here. 2471 */ 2472 cgroup_release_and_wakeup_rmdir(&ptr->css); 2473 } 2474 2475 void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr) 2476 { 2477 __mem_cgroup_commit_charge_swapin(page, ptr, 2478 MEM_CGROUP_CHARGE_TYPE_MAPPED); 2479 } 2480 2481 void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem) 2482 { 2483 if (mem_cgroup_disabled()) 2484 return; 2485 if (!mem) 2486 return; 2487 mem_cgroup_cancel_charge(mem, PAGE_SIZE); 2488 } 2489 2490 static void 2491 __do_uncharge(struct mem_cgroup *mem, const enum charge_type ctype, 2492 int page_size) 2493 { 2494 struct memcg_batch_info *batch = NULL; 2495 bool uncharge_memsw = true; 2496 /* If swapout, usage of swap doesn't decrease */ 2497 if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) 2498 uncharge_memsw = false; 2499 2500 batch = ¤t->memcg_batch; 2501 /* 2502 * In usual, we do css_get() when we remember memcg pointer. 2503 * But in this case, we keep res->usage until end of a series of 2504 * uncharges. Then, it's ok to ignore memcg's refcnt. 2505 */ 2506 if (!batch->memcg) 2507 batch->memcg = mem; 2508 /* 2509 * do_batch > 0 when unmapping pages or inode invalidate/truncate. 2510 * In those cases, all pages freed continously can be expected to be in 2511 * the same cgroup and we have chance to coalesce uncharges. 2512 * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE) 2513 * because we want to do uncharge as soon as possible. 2514 */ 2515 2516 if (!batch->do_batch || test_thread_flag(TIF_MEMDIE)) 2517 goto direct_uncharge; 2518 2519 if (page_size != PAGE_SIZE) 2520 goto direct_uncharge; 2521 2522 /* 2523 * In typical case, batch->memcg == mem. This means we can 2524 * merge a series of uncharges to an uncharge of res_counter. 2525 * If not, we uncharge res_counter ony by one. 2526 */ 2527 if (batch->memcg != mem) 2528 goto direct_uncharge; 2529 /* remember freed charge and uncharge it later */ 2530 batch->bytes += PAGE_SIZE; 2531 if (uncharge_memsw) 2532 batch->memsw_bytes += PAGE_SIZE; 2533 return; 2534 direct_uncharge: 2535 res_counter_uncharge(&mem->res, page_size); 2536 if (uncharge_memsw) 2537 res_counter_uncharge(&mem->memsw, page_size); 2538 if (unlikely(batch->memcg != mem)) 2539 memcg_oom_recover(mem); 2540 return; 2541 } 2542 2543 /* 2544 * uncharge if !page_mapped(page) 2545 */ 2546 static struct mem_cgroup * 2547 __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype) 2548 { 2549 int i; 2550 int count; 2551 struct page_cgroup *pc; 2552 struct mem_cgroup *mem = NULL; 2553 int page_size = PAGE_SIZE; 2554 2555 if (mem_cgroup_disabled()) 2556 return NULL; 2557 2558 if (PageSwapCache(page)) 2559 return NULL; 2560 2561 if (PageTransHuge(page)) { 2562 page_size <<= compound_order(page); 2563 VM_BUG_ON(!PageTransHuge(page)); 2564 } 2565 2566 count = page_size >> PAGE_SHIFT; 2567 /* 2568 * Check if our page_cgroup is valid 2569 */ 2570 pc = lookup_page_cgroup(page); 2571 if (unlikely(!pc || !PageCgroupUsed(pc))) 2572 return NULL; 2573 2574 lock_page_cgroup(pc); 2575 2576 mem = pc->mem_cgroup; 2577 2578 if (!PageCgroupUsed(pc)) 2579 goto unlock_out; 2580 2581 switch (ctype) { 2582 case MEM_CGROUP_CHARGE_TYPE_MAPPED: 2583 case MEM_CGROUP_CHARGE_TYPE_DROP: 2584 /* See mem_cgroup_prepare_migration() */ 2585 if (page_mapped(page) || PageCgroupMigration(pc)) 2586 goto unlock_out; 2587 break; 2588 case MEM_CGROUP_CHARGE_TYPE_SWAPOUT: 2589 if (!PageAnon(page)) { /* Shared memory */ 2590 if (page->mapping && !page_is_file_cache(page)) 2591 goto unlock_out; 2592 } else if (page_mapped(page)) /* Anon */ 2593 goto unlock_out; 2594 break; 2595 default: 2596 break; 2597 } 2598 2599 for (i = 0; i < count; i++) 2600 mem_cgroup_charge_statistics(mem, pc + i, false); 2601 2602 ClearPageCgroupUsed(pc); 2603 /* 2604 * pc->mem_cgroup is not cleared here. It will be accessed when it's 2605 * freed from LRU. This is safe because uncharged page is expected not 2606 * to be reused (freed soon). Exception is SwapCache, it's handled by 2607 * special functions. 2608 */ 2609 2610 unlock_page_cgroup(pc); 2611 /* 2612 * even after unlock, we have mem->res.usage here and this memcg 2613 * will never be freed. 2614 */ 2615 memcg_check_events(mem, page); 2616 if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) { 2617 mem_cgroup_swap_statistics(mem, true); 2618 mem_cgroup_get(mem); 2619 } 2620 if (!mem_cgroup_is_root(mem)) 2621 __do_uncharge(mem, ctype, page_size); 2622 2623 return mem; 2624 2625 unlock_out: 2626 unlock_page_cgroup(pc); 2627 return NULL; 2628 } 2629 2630 void mem_cgroup_uncharge_page(struct page *page) 2631 { 2632 /* early check. */ 2633 if (page_mapped(page)) 2634 return; 2635 if (page->mapping && !PageAnon(page)) 2636 return; 2637 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED); 2638 } 2639 2640 void mem_cgroup_uncharge_cache_page(struct page *page) 2641 { 2642 VM_BUG_ON(page_mapped(page)); 2643 VM_BUG_ON(page->mapping); 2644 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE); 2645 } 2646 2647 /* 2648 * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate. 2649 * In that cases, pages are freed continuously and we can expect pages 2650 * are in the same memcg. All these calls itself limits the number of 2651 * pages freed at once, then uncharge_start/end() is called properly. 2652 * This may be called prural(2) times in a context, 2653 */ 2654 2655 void mem_cgroup_uncharge_start(void) 2656 { 2657 current->memcg_batch.do_batch++; 2658 /* We can do nest. */ 2659 if (current->memcg_batch.do_batch == 1) { 2660 current->memcg_batch.memcg = NULL; 2661 current->memcg_batch.bytes = 0; 2662 current->memcg_batch.memsw_bytes = 0; 2663 } 2664 } 2665 2666 void mem_cgroup_uncharge_end(void) 2667 { 2668 struct memcg_batch_info *batch = ¤t->memcg_batch; 2669 2670 if (!batch->do_batch) 2671 return; 2672 2673 batch->do_batch--; 2674 if (batch->do_batch) /* If stacked, do nothing. */ 2675 return; 2676 2677 if (!batch->memcg) 2678 return; 2679 /* 2680 * This "batch->memcg" is valid without any css_get/put etc... 2681 * bacause we hide charges behind us. 2682 */ 2683 if (batch->bytes) 2684 res_counter_uncharge(&batch->memcg->res, batch->bytes); 2685 if (batch->memsw_bytes) 2686 res_counter_uncharge(&batch->memcg->memsw, batch->memsw_bytes); 2687 memcg_oom_recover(batch->memcg); 2688 /* forget this pointer (for sanity check) */ 2689 batch->memcg = NULL; 2690 } 2691 2692 #ifdef CONFIG_SWAP 2693 /* 2694 * called after __delete_from_swap_cache() and drop "page" account. 2695 * memcg information is recorded to swap_cgroup of "ent" 2696 */ 2697 void 2698 mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout) 2699 { 2700 struct mem_cgroup *memcg; 2701 int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT; 2702 2703 if (!swapout) /* this was a swap cache but the swap is unused ! */ 2704 ctype = MEM_CGROUP_CHARGE_TYPE_DROP; 2705 2706 memcg = __mem_cgroup_uncharge_common(page, ctype); 2707 2708 /* 2709 * record memcg information, if swapout && memcg != NULL, 2710 * mem_cgroup_get() was called in uncharge(). 2711 */ 2712 if (do_swap_account && swapout && memcg) 2713 swap_cgroup_record(ent, css_id(&memcg->css)); 2714 } 2715 #endif 2716 2717 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP 2718 /* 2719 * called from swap_entry_free(). remove record in swap_cgroup and 2720 * uncharge "memsw" account. 2721 */ 2722 void mem_cgroup_uncharge_swap(swp_entry_t ent) 2723 { 2724 struct mem_cgroup *memcg; 2725 unsigned short id; 2726 2727 if (!do_swap_account) 2728 return; 2729 2730 id = swap_cgroup_record(ent, 0); 2731 rcu_read_lock(); 2732 memcg = mem_cgroup_lookup(id); 2733 if (memcg) { 2734 /* 2735 * We uncharge this because swap is freed. 2736 * This memcg can be obsolete one. We avoid calling css_tryget 2737 */ 2738 if (!mem_cgroup_is_root(memcg)) 2739 res_counter_uncharge(&memcg->memsw, PAGE_SIZE); 2740 mem_cgroup_swap_statistics(memcg, false); 2741 mem_cgroup_put(memcg); 2742 } 2743 rcu_read_unlock(); 2744 } 2745 2746 /** 2747 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record. 2748 * @entry: swap entry to be moved 2749 * @from: mem_cgroup which the entry is moved from 2750 * @to: mem_cgroup which the entry is moved to 2751 * @need_fixup: whether we should fixup res_counters and refcounts. 2752 * 2753 * It succeeds only when the swap_cgroup's record for this entry is the same 2754 * as the mem_cgroup's id of @from. 2755 * 2756 * Returns 0 on success, -EINVAL on failure. 2757 * 2758 * The caller must have charged to @to, IOW, called res_counter_charge() about 2759 * both res and memsw, and called css_get(). 2760 */ 2761 static int mem_cgroup_move_swap_account(swp_entry_t entry, 2762 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup) 2763 { 2764 unsigned short old_id, new_id; 2765 2766 old_id = css_id(&from->css); 2767 new_id = css_id(&to->css); 2768 2769 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) { 2770 mem_cgroup_swap_statistics(from, false); 2771 mem_cgroup_swap_statistics(to, true); 2772 /* 2773 * This function is only called from task migration context now. 2774 * It postpones res_counter and refcount handling till the end 2775 * of task migration(mem_cgroup_clear_mc()) for performance 2776 * improvement. But we cannot postpone mem_cgroup_get(to) 2777 * because if the process that has been moved to @to does 2778 * swap-in, the refcount of @to might be decreased to 0. 2779 */ 2780 mem_cgroup_get(to); 2781 if (need_fixup) { 2782 if (!mem_cgroup_is_root(from)) 2783 res_counter_uncharge(&from->memsw, PAGE_SIZE); 2784 mem_cgroup_put(from); 2785 /* 2786 * we charged both to->res and to->memsw, so we should 2787 * uncharge to->res. 2788 */ 2789 if (!mem_cgroup_is_root(to)) 2790 res_counter_uncharge(&to->res, PAGE_SIZE); 2791 } 2792 return 0; 2793 } 2794 return -EINVAL; 2795 } 2796 #else 2797 static inline int mem_cgroup_move_swap_account(swp_entry_t entry, 2798 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup) 2799 { 2800 return -EINVAL; 2801 } 2802 #endif 2803 2804 /* 2805 * Before starting migration, account PAGE_SIZE to mem_cgroup that the old 2806 * page belongs to. 2807 */ 2808 int mem_cgroup_prepare_migration(struct page *page, 2809 struct page *newpage, struct mem_cgroup **ptr) 2810 { 2811 struct page_cgroup *pc; 2812 struct mem_cgroup *mem = NULL; 2813 enum charge_type ctype; 2814 int ret = 0; 2815 2816 VM_BUG_ON(PageTransHuge(page)); 2817 if (mem_cgroup_disabled()) 2818 return 0; 2819 2820 pc = lookup_page_cgroup(page); 2821 lock_page_cgroup(pc); 2822 if (PageCgroupUsed(pc)) { 2823 mem = pc->mem_cgroup; 2824 css_get(&mem->css); 2825 /* 2826 * At migrating an anonymous page, its mapcount goes down 2827 * to 0 and uncharge() will be called. But, even if it's fully 2828 * unmapped, migration may fail and this page has to be 2829 * charged again. We set MIGRATION flag here and delay uncharge 2830 * until end_migration() is called 2831 * 2832 * Corner Case Thinking 2833 * A) 2834 * When the old page was mapped as Anon and it's unmap-and-freed 2835 * while migration was ongoing. 2836 * If unmap finds the old page, uncharge() of it will be delayed 2837 * until end_migration(). If unmap finds a new page, it's 2838 * uncharged when it make mapcount to be 1->0. If unmap code 2839 * finds swap_migration_entry, the new page will not be mapped 2840 * and end_migration() will find it(mapcount==0). 2841 * 2842 * B) 2843 * When the old page was mapped but migraion fails, the kernel 2844 * remaps it. A charge for it is kept by MIGRATION flag even 2845 * if mapcount goes down to 0. We can do remap successfully 2846 * without charging it again. 2847 * 2848 * C) 2849 * The "old" page is under lock_page() until the end of 2850 * migration, so, the old page itself will not be swapped-out. 2851 * If the new page is swapped out before end_migraton, our 2852 * hook to usual swap-out path will catch the event. 2853 */ 2854 if (PageAnon(page)) 2855 SetPageCgroupMigration(pc); 2856 } 2857 unlock_page_cgroup(pc); 2858 /* 2859 * If the page is not charged at this point, 2860 * we return here. 2861 */ 2862 if (!mem) 2863 return 0; 2864 2865 *ptr = mem; 2866 ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, ptr, false, PAGE_SIZE); 2867 css_put(&mem->css);/* drop extra refcnt */ 2868 if (ret || *ptr == NULL) { 2869 if (PageAnon(page)) { 2870 lock_page_cgroup(pc); 2871 ClearPageCgroupMigration(pc); 2872 unlock_page_cgroup(pc); 2873 /* 2874 * The old page may be fully unmapped while we kept it. 2875 */ 2876 mem_cgroup_uncharge_page(page); 2877 } 2878 return -ENOMEM; 2879 } 2880 /* 2881 * We charge new page before it's used/mapped. So, even if unlock_page() 2882 * is called before end_migration, we can catch all events on this new 2883 * page. In the case new page is migrated but not remapped, new page's 2884 * mapcount will be finally 0 and we call uncharge in end_migration(). 2885 */ 2886 pc = lookup_page_cgroup(newpage); 2887 if (PageAnon(page)) 2888 ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED; 2889 else if (page_is_file_cache(page)) 2890 ctype = MEM_CGROUP_CHARGE_TYPE_CACHE; 2891 else 2892 ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM; 2893 __mem_cgroup_commit_charge(mem, pc, ctype, PAGE_SIZE); 2894 return ret; 2895 } 2896 2897 /* remove redundant charge if migration failed*/ 2898 void mem_cgroup_end_migration(struct mem_cgroup *mem, 2899 struct page *oldpage, struct page *newpage, bool migration_ok) 2900 { 2901 struct page *used, *unused; 2902 struct page_cgroup *pc; 2903 2904 if (!mem) 2905 return; 2906 /* blocks rmdir() */ 2907 cgroup_exclude_rmdir(&mem->css); 2908 if (!migration_ok) { 2909 used = oldpage; 2910 unused = newpage; 2911 } else { 2912 used = newpage; 2913 unused = oldpage; 2914 } 2915 /* 2916 * We disallowed uncharge of pages under migration because mapcount 2917 * of the page goes down to zero, temporarly. 2918 * Clear the flag and check the page should be charged. 2919 */ 2920 pc = lookup_page_cgroup(oldpage); 2921 lock_page_cgroup(pc); 2922 ClearPageCgroupMigration(pc); 2923 unlock_page_cgroup(pc); 2924 2925 __mem_cgroup_uncharge_common(unused, MEM_CGROUP_CHARGE_TYPE_FORCE); 2926 2927 /* 2928 * If a page is a file cache, radix-tree replacement is very atomic 2929 * and we can skip this check. When it was an Anon page, its mapcount 2930 * goes down to 0. But because we added MIGRATION flage, it's not 2931 * uncharged yet. There are several case but page->mapcount check 2932 * and USED bit check in mem_cgroup_uncharge_page() will do enough 2933 * check. (see prepare_charge() also) 2934 */ 2935 if (PageAnon(used)) 2936 mem_cgroup_uncharge_page(used); 2937 /* 2938 * At migration, we may charge account against cgroup which has no 2939 * tasks. 2940 * So, rmdir()->pre_destroy() can be called while we do this charge. 2941 * In that case, we need to call pre_destroy() again. check it here. 2942 */ 2943 cgroup_release_and_wakeup_rmdir(&mem->css); 2944 } 2945 2946 /* 2947 * A call to try to shrink memory usage on charge failure at shmem's swapin. 2948 * Calling hierarchical_reclaim is not enough because we should update 2949 * last_oom_jiffies to prevent pagefault_out_of_memory from invoking global OOM. 2950 * Moreover considering hierarchy, we should reclaim from the mem_over_limit, 2951 * not from the memcg which this page would be charged to. 2952 * try_charge_swapin does all of these works properly. 2953 */ 2954 int mem_cgroup_shmem_charge_fallback(struct page *page, 2955 struct mm_struct *mm, 2956 gfp_t gfp_mask) 2957 { 2958 struct mem_cgroup *mem = NULL; 2959 int ret; 2960 2961 if (mem_cgroup_disabled()) 2962 return 0; 2963 2964 ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem); 2965 if (!ret) 2966 mem_cgroup_cancel_charge_swapin(mem); /* it does !mem check */ 2967 2968 return ret; 2969 } 2970 2971 static DEFINE_MUTEX(set_limit_mutex); 2972 2973 static int mem_cgroup_resize_limit(struct mem_cgroup *memcg, 2974 unsigned long long val) 2975 { 2976 int retry_count; 2977 u64 memswlimit, memlimit; 2978 int ret = 0; 2979 int children = mem_cgroup_count_children(memcg); 2980 u64 curusage, oldusage; 2981 int enlarge; 2982 2983 /* 2984 * For keeping hierarchical_reclaim simple, how long we should retry 2985 * is depends on callers. We set our retry-count to be function 2986 * of # of children which we should visit in this loop. 2987 */ 2988 retry_count = MEM_CGROUP_RECLAIM_RETRIES * children; 2989 2990 oldusage = res_counter_read_u64(&memcg->res, RES_USAGE); 2991 2992 enlarge = 0; 2993 while (retry_count) { 2994 if (signal_pending(current)) { 2995 ret = -EINTR; 2996 break; 2997 } 2998 /* 2999 * Rather than hide all in some function, I do this in 3000 * open coded manner. You see what this really does. 3001 * We have to guarantee mem->res.limit < mem->memsw.limit. 3002 */ 3003 mutex_lock(&set_limit_mutex); 3004 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 3005 if (memswlimit < val) { 3006 ret = -EINVAL; 3007 mutex_unlock(&set_limit_mutex); 3008 break; 3009 } 3010 3011 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT); 3012 if (memlimit < val) 3013 enlarge = 1; 3014 3015 ret = res_counter_set_limit(&memcg->res, val); 3016 if (!ret) { 3017 if (memswlimit == val) 3018 memcg->memsw_is_minimum = true; 3019 else 3020 memcg->memsw_is_minimum = false; 3021 } 3022 mutex_unlock(&set_limit_mutex); 3023 3024 if (!ret) 3025 break; 3026 3027 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL, 3028 MEM_CGROUP_RECLAIM_SHRINK); 3029 curusage = res_counter_read_u64(&memcg->res, RES_USAGE); 3030 /* Usage is reduced ? */ 3031 if (curusage >= oldusage) 3032 retry_count--; 3033 else 3034 oldusage = curusage; 3035 } 3036 if (!ret && enlarge) 3037 memcg_oom_recover(memcg); 3038 3039 return ret; 3040 } 3041 3042 static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg, 3043 unsigned long long val) 3044 { 3045 int retry_count; 3046 u64 memlimit, memswlimit, oldusage, curusage; 3047 int children = mem_cgroup_count_children(memcg); 3048 int ret = -EBUSY; 3049 int enlarge = 0; 3050 3051 /* see mem_cgroup_resize_res_limit */ 3052 retry_count = children * MEM_CGROUP_RECLAIM_RETRIES; 3053 oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE); 3054 while (retry_count) { 3055 if (signal_pending(current)) { 3056 ret = -EINTR; 3057 break; 3058 } 3059 /* 3060 * Rather than hide all in some function, I do this in 3061 * open coded manner. You see what this really does. 3062 * We have to guarantee mem->res.limit < mem->memsw.limit. 3063 */ 3064 mutex_lock(&set_limit_mutex); 3065 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT); 3066 if (memlimit > val) { 3067 ret = -EINVAL; 3068 mutex_unlock(&set_limit_mutex); 3069 break; 3070 } 3071 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 3072 if (memswlimit < val) 3073 enlarge = 1; 3074 ret = res_counter_set_limit(&memcg->memsw, val); 3075 if (!ret) { 3076 if (memlimit == val) 3077 memcg->memsw_is_minimum = true; 3078 else 3079 memcg->memsw_is_minimum = false; 3080 } 3081 mutex_unlock(&set_limit_mutex); 3082 3083 if (!ret) 3084 break; 3085 3086 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL, 3087 MEM_CGROUP_RECLAIM_NOSWAP | 3088 MEM_CGROUP_RECLAIM_SHRINK); 3089 curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE); 3090 /* Usage is reduced ? */ 3091 if (curusage >= oldusage) 3092 retry_count--; 3093 else 3094 oldusage = curusage; 3095 } 3096 if (!ret && enlarge) 3097 memcg_oom_recover(memcg); 3098 return ret; 3099 } 3100 3101 unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order, 3102 gfp_t gfp_mask) 3103 { 3104 unsigned long nr_reclaimed = 0; 3105 struct mem_cgroup_per_zone *mz, *next_mz = NULL; 3106 unsigned long reclaimed; 3107 int loop = 0; 3108 struct mem_cgroup_tree_per_zone *mctz; 3109 unsigned long long excess; 3110 3111 if (order > 0) 3112 return 0; 3113 3114 mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone)); 3115 /* 3116 * This loop can run a while, specially if mem_cgroup's continuously 3117 * keep exceeding their soft limit and putting the system under 3118 * pressure 3119 */ 3120 do { 3121 if (next_mz) 3122 mz = next_mz; 3123 else 3124 mz = mem_cgroup_largest_soft_limit_node(mctz); 3125 if (!mz) 3126 break; 3127 3128 reclaimed = mem_cgroup_hierarchical_reclaim(mz->mem, zone, 3129 gfp_mask, 3130 MEM_CGROUP_RECLAIM_SOFT); 3131 nr_reclaimed += reclaimed; 3132 spin_lock(&mctz->lock); 3133 3134 /* 3135 * If we failed to reclaim anything from this memory cgroup 3136 * it is time to move on to the next cgroup 3137 */ 3138 next_mz = NULL; 3139 if (!reclaimed) { 3140 do { 3141 /* 3142 * Loop until we find yet another one. 3143 * 3144 * By the time we get the soft_limit lock 3145 * again, someone might have aded the 3146 * group back on the RB tree. Iterate to 3147 * make sure we get a different mem. 3148 * mem_cgroup_largest_soft_limit_node returns 3149 * NULL if no other cgroup is present on 3150 * the tree 3151 */ 3152 next_mz = 3153 __mem_cgroup_largest_soft_limit_node(mctz); 3154 if (next_mz == mz) { 3155 css_put(&next_mz->mem->css); 3156 next_mz = NULL; 3157 } else /* next_mz == NULL or other memcg */ 3158 break; 3159 } while (1); 3160 } 3161 __mem_cgroup_remove_exceeded(mz->mem, mz, mctz); 3162 excess = res_counter_soft_limit_excess(&mz->mem->res); 3163 /* 3164 * One school of thought says that we should not add 3165 * back the node to the tree if reclaim returns 0. 3166 * But our reclaim could return 0, simply because due 3167 * to priority we are exposing a smaller subset of 3168 * memory to reclaim from. Consider this as a longer 3169 * term TODO. 3170 */ 3171 /* If excess == 0, no tree ops */ 3172 __mem_cgroup_insert_exceeded(mz->mem, mz, mctz, excess); 3173 spin_unlock(&mctz->lock); 3174 css_put(&mz->mem->css); 3175 loop++; 3176 /* 3177 * Could not reclaim anything and there are no more 3178 * mem cgroups to try or we seem to be looping without 3179 * reclaiming anything. 3180 */ 3181 if (!nr_reclaimed && 3182 (next_mz == NULL || 3183 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS)) 3184 break; 3185 } while (!nr_reclaimed); 3186 if (next_mz) 3187 css_put(&next_mz->mem->css); 3188 return nr_reclaimed; 3189 } 3190 3191 /* 3192 * This routine traverse page_cgroup in given list and drop them all. 3193 * *And* this routine doesn't reclaim page itself, just removes page_cgroup. 3194 */ 3195 static int mem_cgroup_force_empty_list(struct mem_cgroup *mem, 3196 int node, int zid, enum lru_list lru) 3197 { 3198 struct zone *zone; 3199 struct mem_cgroup_per_zone *mz; 3200 struct page_cgroup *pc, *busy; 3201 unsigned long flags, loop; 3202 struct list_head *list; 3203 int ret = 0; 3204 3205 zone = &NODE_DATA(node)->node_zones[zid]; 3206 mz = mem_cgroup_zoneinfo(mem, node, zid); 3207 list = &mz->lists[lru]; 3208 3209 loop = MEM_CGROUP_ZSTAT(mz, lru); 3210 /* give some margin against EBUSY etc...*/ 3211 loop += 256; 3212 busy = NULL; 3213 while (loop--) { 3214 ret = 0; 3215 spin_lock_irqsave(&zone->lru_lock, flags); 3216 if (list_empty(list)) { 3217 spin_unlock_irqrestore(&zone->lru_lock, flags); 3218 break; 3219 } 3220 pc = list_entry(list->prev, struct page_cgroup, lru); 3221 if (busy == pc) { 3222 list_move(&pc->lru, list); 3223 busy = NULL; 3224 spin_unlock_irqrestore(&zone->lru_lock, flags); 3225 continue; 3226 } 3227 spin_unlock_irqrestore(&zone->lru_lock, flags); 3228 3229 ret = mem_cgroup_move_parent(pc, mem, GFP_KERNEL); 3230 if (ret == -ENOMEM) 3231 break; 3232 3233 if (ret == -EBUSY || ret == -EINVAL) { 3234 /* found lock contention or "pc" is obsolete. */ 3235 busy = pc; 3236 cond_resched(); 3237 } else 3238 busy = NULL; 3239 } 3240 3241 if (!ret && !list_empty(list)) 3242 return -EBUSY; 3243 return ret; 3244 } 3245 3246 /* 3247 * make mem_cgroup's charge to be 0 if there is no task. 3248 * This enables deleting this mem_cgroup. 3249 */ 3250 static int mem_cgroup_force_empty(struct mem_cgroup *mem, bool free_all) 3251 { 3252 int ret; 3253 int node, zid, shrink; 3254 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES; 3255 struct cgroup *cgrp = mem->css.cgroup; 3256 3257 css_get(&mem->css); 3258 3259 shrink = 0; 3260 /* should free all ? */ 3261 if (free_all) 3262 goto try_to_free; 3263 move_account: 3264 do { 3265 ret = -EBUSY; 3266 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children)) 3267 goto out; 3268 ret = -EINTR; 3269 if (signal_pending(current)) 3270 goto out; 3271 /* This is for making all *used* pages to be on LRU. */ 3272 lru_add_drain_all(); 3273 drain_all_stock_sync(); 3274 ret = 0; 3275 mem_cgroup_start_move(mem); 3276 for_each_node_state(node, N_HIGH_MEMORY) { 3277 for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) { 3278 enum lru_list l; 3279 for_each_lru(l) { 3280 ret = mem_cgroup_force_empty_list(mem, 3281 node, zid, l); 3282 if (ret) 3283 break; 3284 } 3285 } 3286 if (ret) 3287 break; 3288 } 3289 mem_cgroup_end_move(mem); 3290 memcg_oom_recover(mem); 3291 /* it seems parent cgroup doesn't have enough mem */ 3292 if (ret == -ENOMEM) 3293 goto try_to_free; 3294 cond_resched(); 3295 /* "ret" should also be checked to ensure all lists are empty. */ 3296 } while (mem->res.usage > 0 || ret); 3297 out: 3298 css_put(&mem->css); 3299 return ret; 3300 3301 try_to_free: 3302 /* returns EBUSY if there is a task or if we come here twice. */ 3303 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children) || shrink) { 3304 ret = -EBUSY; 3305 goto out; 3306 } 3307 /* we call try-to-free pages for make this cgroup empty */ 3308 lru_add_drain_all(); 3309 /* try to free all pages in this cgroup */ 3310 shrink = 1; 3311 while (nr_retries && mem->res.usage > 0) { 3312 int progress; 3313 3314 if (signal_pending(current)) { 3315 ret = -EINTR; 3316 goto out; 3317 } 3318 progress = try_to_free_mem_cgroup_pages(mem, GFP_KERNEL, 3319 false, get_swappiness(mem)); 3320 if (!progress) { 3321 nr_retries--; 3322 /* maybe some writeback is necessary */ 3323 congestion_wait(BLK_RW_ASYNC, HZ/10); 3324 } 3325 3326 } 3327 lru_add_drain(); 3328 /* try move_account...there may be some *locked* pages. */ 3329 goto move_account; 3330 } 3331 3332 int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event) 3333 { 3334 return mem_cgroup_force_empty(mem_cgroup_from_cont(cont), true); 3335 } 3336 3337 3338 static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft) 3339 { 3340 return mem_cgroup_from_cont(cont)->use_hierarchy; 3341 } 3342 3343 static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft, 3344 u64 val) 3345 { 3346 int retval = 0; 3347 struct mem_cgroup *mem = mem_cgroup_from_cont(cont); 3348 struct cgroup *parent = cont->parent; 3349 struct mem_cgroup *parent_mem = NULL; 3350 3351 if (parent) 3352 parent_mem = mem_cgroup_from_cont(parent); 3353 3354 cgroup_lock(); 3355 /* 3356 * If parent's use_hierarchy is set, we can't make any modifications 3357 * in the child subtrees. If it is unset, then the change can 3358 * occur, provided the current cgroup has no children. 3359 * 3360 * For the root cgroup, parent_mem is NULL, we allow value to be 3361 * set if there are no children. 3362 */ 3363 if ((!parent_mem || !parent_mem->use_hierarchy) && 3364 (val == 1 || val == 0)) { 3365 if (list_empty(&cont->children)) 3366 mem->use_hierarchy = val; 3367 else 3368 retval = -EBUSY; 3369 } else 3370 retval = -EINVAL; 3371 cgroup_unlock(); 3372 3373 return retval; 3374 } 3375 3376 3377 static u64 mem_cgroup_get_recursive_idx_stat(struct mem_cgroup *mem, 3378 enum mem_cgroup_stat_index idx) 3379 { 3380 struct mem_cgroup *iter; 3381 s64 val = 0; 3382 3383 /* each per cpu's value can be minus.Then, use s64 */ 3384 for_each_mem_cgroup_tree(iter, mem) 3385 val += mem_cgroup_read_stat(iter, idx); 3386 3387 if (val < 0) /* race ? */ 3388 val = 0; 3389 return val; 3390 } 3391 3392 static inline u64 mem_cgroup_usage(struct mem_cgroup *mem, bool swap) 3393 { 3394 u64 val; 3395 3396 if (!mem_cgroup_is_root(mem)) { 3397 if (!swap) 3398 return res_counter_read_u64(&mem->res, RES_USAGE); 3399 else 3400 return res_counter_read_u64(&mem->memsw, RES_USAGE); 3401 } 3402 3403 val = mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_CACHE); 3404 val += mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_RSS); 3405 3406 if (swap) 3407 val += mem_cgroup_get_recursive_idx_stat(mem, 3408 MEM_CGROUP_STAT_SWAPOUT); 3409 3410 return val << PAGE_SHIFT; 3411 } 3412 3413 static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft) 3414 { 3415 struct mem_cgroup *mem = mem_cgroup_from_cont(cont); 3416 u64 val; 3417 int type, name; 3418 3419 type = MEMFILE_TYPE(cft->private); 3420 name = MEMFILE_ATTR(cft->private); 3421 switch (type) { 3422 case _MEM: 3423 if (name == RES_USAGE) 3424 val = mem_cgroup_usage(mem, false); 3425 else 3426 val = res_counter_read_u64(&mem->res, name); 3427 break; 3428 case _MEMSWAP: 3429 if (name == RES_USAGE) 3430 val = mem_cgroup_usage(mem, true); 3431 else 3432 val = res_counter_read_u64(&mem->memsw, name); 3433 break; 3434 default: 3435 BUG(); 3436 break; 3437 } 3438 return val; 3439 } 3440 /* 3441 * The user of this function is... 3442 * RES_LIMIT. 3443 */ 3444 static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft, 3445 const char *buffer) 3446 { 3447 struct mem_cgroup *memcg = mem_cgroup_from_cont(cont); 3448 int type, name; 3449 unsigned long long val; 3450 int ret; 3451 3452 type = MEMFILE_TYPE(cft->private); 3453 name = MEMFILE_ATTR(cft->private); 3454 switch (name) { 3455 case RES_LIMIT: 3456 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */ 3457 ret = -EINVAL; 3458 break; 3459 } 3460 /* This function does all necessary parse...reuse it */ 3461 ret = res_counter_memparse_write_strategy(buffer, &val); 3462 if (ret) 3463 break; 3464 if (type == _MEM) 3465 ret = mem_cgroup_resize_limit(memcg, val); 3466 else 3467 ret = mem_cgroup_resize_memsw_limit(memcg, val); 3468 break; 3469 case RES_SOFT_LIMIT: 3470 ret = res_counter_memparse_write_strategy(buffer, &val); 3471 if (ret) 3472 break; 3473 /* 3474 * For memsw, soft limits are hard to implement in terms 3475 * of semantics, for now, we support soft limits for 3476 * control without swap 3477 */ 3478 if (type == _MEM) 3479 ret = res_counter_set_soft_limit(&memcg->res, val); 3480 else 3481 ret = -EINVAL; 3482 break; 3483 default: 3484 ret = -EINVAL; /* should be BUG() ? */ 3485 break; 3486 } 3487 return ret; 3488 } 3489 3490 static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg, 3491 unsigned long long *mem_limit, unsigned long long *memsw_limit) 3492 { 3493 struct cgroup *cgroup; 3494 unsigned long long min_limit, min_memsw_limit, tmp; 3495 3496 min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT); 3497 min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 3498 cgroup = memcg->css.cgroup; 3499 if (!memcg->use_hierarchy) 3500 goto out; 3501 3502 while (cgroup->parent) { 3503 cgroup = cgroup->parent; 3504 memcg = mem_cgroup_from_cont(cgroup); 3505 if (!memcg->use_hierarchy) 3506 break; 3507 tmp = res_counter_read_u64(&memcg->res, RES_LIMIT); 3508 min_limit = min(min_limit, tmp); 3509 tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT); 3510 min_memsw_limit = min(min_memsw_limit, tmp); 3511 } 3512 out: 3513 *mem_limit = min_limit; 3514 *memsw_limit = min_memsw_limit; 3515 return; 3516 } 3517 3518 static int mem_cgroup_reset(struct cgroup *cont, unsigned int event) 3519 { 3520 struct mem_cgroup *mem; 3521 int type, name; 3522 3523 mem = mem_cgroup_from_cont(cont); 3524 type = MEMFILE_TYPE(event); 3525 name = MEMFILE_ATTR(event); 3526 switch (name) { 3527 case RES_MAX_USAGE: 3528 if (type == _MEM) 3529 res_counter_reset_max(&mem->res); 3530 else 3531 res_counter_reset_max(&mem->memsw); 3532 break; 3533 case RES_FAILCNT: 3534 if (type == _MEM) 3535 res_counter_reset_failcnt(&mem->res); 3536 else 3537 res_counter_reset_failcnt(&mem->memsw); 3538 break; 3539 } 3540 3541 return 0; 3542 } 3543 3544 static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp, 3545 struct cftype *cft) 3546 { 3547 return mem_cgroup_from_cont(cgrp)->move_charge_at_immigrate; 3548 } 3549 3550 #ifdef CONFIG_MMU 3551 static int mem_cgroup_move_charge_write(struct cgroup *cgrp, 3552 struct cftype *cft, u64 val) 3553 { 3554 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp); 3555 3556 if (val >= (1 << NR_MOVE_TYPE)) 3557 return -EINVAL; 3558 /* 3559 * We check this value several times in both in can_attach() and 3560 * attach(), so we need cgroup lock to prevent this value from being 3561 * inconsistent. 3562 */ 3563 cgroup_lock(); 3564 mem->move_charge_at_immigrate = val; 3565 cgroup_unlock(); 3566 3567 return 0; 3568 } 3569 #else 3570 static int mem_cgroup_move_charge_write(struct cgroup *cgrp, 3571 struct cftype *cft, u64 val) 3572 { 3573 return -ENOSYS; 3574 } 3575 #endif 3576 3577 3578 /* For read statistics */ 3579 enum { 3580 MCS_CACHE, 3581 MCS_RSS, 3582 MCS_FILE_MAPPED, 3583 MCS_PGPGIN, 3584 MCS_PGPGOUT, 3585 MCS_SWAP, 3586 MCS_INACTIVE_ANON, 3587 MCS_ACTIVE_ANON, 3588 MCS_INACTIVE_FILE, 3589 MCS_ACTIVE_FILE, 3590 MCS_UNEVICTABLE, 3591 NR_MCS_STAT, 3592 }; 3593 3594 struct mcs_total_stat { 3595 s64 stat[NR_MCS_STAT]; 3596 }; 3597 3598 struct { 3599 char *local_name; 3600 char *total_name; 3601 } memcg_stat_strings[NR_MCS_STAT] = { 3602 {"cache", "total_cache"}, 3603 {"rss", "total_rss"}, 3604 {"mapped_file", "total_mapped_file"}, 3605 {"pgpgin", "total_pgpgin"}, 3606 {"pgpgout", "total_pgpgout"}, 3607 {"swap", "total_swap"}, 3608 {"inactive_anon", "total_inactive_anon"}, 3609 {"active_anon", "total_active_anon"}, 3610 {"inactive_file", "total_inactive_file"}, 3611 {"active_file", "total_active_file"}, 3612 {"unevictable", "total_unevictable"} 3613 }; 3614 3615 3616 static void 3617 mem_cgroup_get_local_stat(struct mem_cgroup *mem, struct mcs_total_stat *s) 3618 { 3619 s64 val; 3620 3621 /* per cpu stat */ 3622 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE); 3623 s->stat[MCS_CACHE] += val * PAGE_SIZE; 3624 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS); 3625 s->stat[MCS_RSS] += val * PAGE_SIZE; 3626 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_FILE_MAPPED); 3627 s->stat[MCS_FILE_MAPPED] += val * PAGE_SIZE; 3628 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGIN_COUNT); 3629 s->stat[MCS_PGPGIN] += val; 3630 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGOUT_COUNT); 3631 s->stat[MCS_PGPGOUT] += val; 3632 if (do_swap_account) { 3633 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_SWAPOUT); 3634 s->stat[MCS_SWAP] += val * PAGE_SIZE; 3635 } 3636 3637 /* per zone stat */ 3638 val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_ANON); 3639 s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE; 3640 val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_ANON); 3641 s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE; 3642 val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_FILE); 3643 s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE; 3644 val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_FILE); 3645 s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE; 3646 val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE); 3647 s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE; 3648 } 3649 3650 static void 3651 mem_cgroup_get_total_stat(struct mem_cgroup *mem, struct mcs_total_stat *s) 3652 { 3653 struct mem_cgroup *iter; 3654 3655 for_each_mem_cgroup_tree(iter, mem) 3656 mem_cgroup_get_local_stat(iter, s); 3657 } 3658 3659 static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft, 3660 struct cgroup_map_cb *cb) 3661 { 3662 struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont); 3663 struct mcs_total_stat mystat; 3664 int i; 3665 3666 memset(&mystat, 0, sizeof(mystat)); 3667 mem_cgroup_get_local_stat(mem_cont, &mystat); 3668 3669 for (i = 0; i < NR_MCS_STAT; i++) { 3670 if (i == MCS_SWAP && !do_swap_account) 3671 continue; 3672 cb->fill(cb, memcg_stat_strings[i].local_name, mystat.stat[i]); 3673 } 3674 3675 /* Hierarchical information */ 3676 { 3677 unsigned long long limit, memsw_limit; 3678 memcg_get_hierarchical_limit(mem_cont, &limit, &memsw_limit); 3679 cb->fill(cb, "hierarchical_memory_limit", limit); 3680 if (do_swap_account) 3681 cb->fill(cb, "hierarchical_memsw_limit", memsw_limit); 3682 } 3683 3684 memset(&mystat, 0, sizeof(mystat)); 3685 mem_cgroup_get_total_stat(mem_cont, &mystat); 3686 for (i = 0; i < NR_MCS_STAT; i++) { 3687 if (i == MCS_SWAP && !do_swap_account) 3688 continue; 3689 cb->fill(cb, memcg_stat_strings[i].total_name, mystat.stat[i]); 3690 } 3691 3692 #ifdef CONFIG_DEBUG_VM 3693 cb->fill(cb, "inactive_ratio", calc_inactive_ratio(mem_cont, NULL)); 3694 3695 { 3696 int nid, zid; 3697 struct mem_cgroup_per_zone *mz; 3698 unsigned long recent_rotated[2] = {0, 0}; 3699 unsigned long recent_scanned[2] = {0, 0}; 3700 3701 for_each_online_node(nid) 3702 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 3703 mz = mem_cgroup_zoneinfo(mem_cont, nid, zid); 3704 3705 recent_rotated[0] += 3706 mz->reclaim_stat.recent_rotated[0]; 3707 recent_rotated[1] += 3708 mz->reclaim_stat.recent_rotated[1]; 3709 recent_scanned[0] += 3710 mz->reclaim_stat.recent_scanned[0]; 3711 recent_scanned[1] += 3712 mz->reclaim_stat.recent_scanned[1]; 3713 } 3714 cb->fill(cb, "recent_rotated_anon", recent_rotated[0]); 3715 cb->fill(cb, "recent_rotated_file", recent_rotated[1]); 3716 cb->fill(cb, "recent_scanned_anon", recent_scanned[0]); 3717 cb->fill(cb, "recent_scanned_file", recent_scanned[1]); 3718 } 3719 #endif 3720 3721 return 0; 3722 } 3723 3724 static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft) 3725 { 3726 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); 3727 3728 return get_swappiness(memcg); 3729 } 3730 3731 static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft, 3732 u64 val) 3733 { 3734 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); 3735 struct mem_cgroup *parent; 3736 3737 if (val > 100) 3738 return -EINVAL; 3739 3740 if (cgrp->parent == NULL) 3741 return -EINVAL; 3742 3743 parent = mem_cgroup_from_cont(cgrp->parent); 3744 3745 cgroup_lock(); 3746 3747 /* If under hierarchy, only empty-root can set this value */ 3748 if ((parent->use_hierarchy) || 3749 (memcg->use_hierarchy && !list_empty(&cgrp->children))) { 3750 cgroup_unlock(); 3751 return -EINVAL; 3752 } 3753 3754 spin_lock(&memcg->reclaim_param_lock); 3755 memcg->swappiness = val; 3756 spin_unlock(&memcg->reclaim_param_lock); 3757 3758 cgroup_unlock(); 3759 3760 return 0; 3761 } 3762 3763 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap) 3764 { 3765 struct mem_cgroup_threshold_ary *t; 3766 u64 usage; 3767 int i; 3768 3769 rcu_read_lock(); 3770 if (!swap) 3771 t = rcu_dereference(memcg->thresholds.primary); 3772 else 3773 t = rcu_dereference(memcg->memsw_thresholds.primary); 3774 3775 if (!t) 3776 goto unlock; 3777 3778 usage = mem_cgroup_usage(memcg, swap); 3779 3780 /* 3781 * current_threshold points to threshold just below usage. 3782 * If it's not true, a threshold was crossed after last 3783 * call of __mem_cgroup_threshold(). 3784 */ 3785 i = t->current_threshold; 3786 3787 /* 3788 * Iterate backward over array of thresholds starting from 3789 * current_threshold and check if a threshold is crossed. 3790 * If none of thresholds below usage is crossed, we read 3791 * only one element of the array here. 3792 */ 3793 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--) 3794 eventfd_signal(t->entries[i].eventfd, 1); 3795 3796 /* i = current_threshold + 1 */ 3797 i++; 3798 3799 /* 3800 * Iterate forward over array of thresholds starting from 3801 * current_threshold+1 and check if a threshold is crossed. 3802 * If none of thresholds above usage is crossed, we read 3803 * only one element of the array here. 3804 */ 3805 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++) 3806 eventfd_signal(t->entries[i].eventfd, 1); 3807 3808 /* Update current_threshold */ 3809 t->current_threshold = i - 1; 3810 unlock: 3811 rcu_read_unlock(); 3812 } 3813 3814 static void mem_cgroup_threshold(struct mem_cgroup *memcg) 3815 { 3816 while (memcg) { 3817 __mem_cgroup_threshold(memcg, false); 3818 if (do_swap_account) 3819 __mem_cgroup_threshold(memcg, true); 3820 3821 memcg = parent_mem_cgroup(memcg); 3822 } 3823 } 3824 3825 static int compare_thresholds(const void *a, const void *b) 3826 { 3827 const struct mem_cgroup_threshold *_a = a; 3828 const struct mem_cgroup_threshold *_b = b; 3829 3830 return _a->threshold - _b->threshold; 3831 } 3832 3833 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *mem) 3834 { 3835 struct mem_cgroup_eventfd_list *ev; 3836 3837 list_for_each_entry(ev, &mem->oom_notify, list) 3838 eventfd_signal(ev->eventfd, 1); 3839 return 0; 3840 } 3841 3842 static void mem_cgroup_oom_notify(struct mem_cgroup *mem) 3843 { 3844 struct mem_cgroup *iter; 3845 3846 for_each_mem_cgroup_tree(iter, mem) 3847 mem_cgroup_oom_notify_cb(iter); 3848 } 3849 3850 static int mem_cgroup_usage_register_event(struct cgroup *cgrp, 3851 struct cftype *cft, struct eventfd_ctx *eventfd, const char *args) 3852 { 3853 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); 3854 struct mem_cgroup_thresholds *thresholds; 3855 struct mem_cgroup_threshold_ary *new; 3856 int type = MEMFILE_TYPE(cft->private); 3857 u64 threshold, usage; 3858 int i, size, ret; 3859 3860 ret = res_counter_memparse_write_strategy(args, &threshold); 3861 if (ret) 3862 return ret; 3863 3864 mutex_lock(&memcg->thresholds_lock); 3865 3866 if (type == _MEM) 3867 thresholds = &memcg->thresholds; 3868 else if (type == _MEMSWAP) 3869 thresholds = &memcg->memsw_thresholds; 3870 else 3871 BUG(); 3872 3873 usage = mem_cgroup_usage(memcg, type == _MEMSWAP); 3874 3875 /* Check if a threshold crossed before adding a new one */ 3876 if (thresholds->primary) 3877 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 3878 3879 size = thresholds->primary ? thresholds->primary->size + 1 : 1; 3880 3881 /* Allocate memory for new array of thresholds */ 3882 new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold), 3883 GFP_KERNEL); 3884 if (!new) { 3885 ret = -ENOMEM; 3886 goto unlock; 3887 } 3888 new->size = size; 3889 3890 /* Copy thresholds (if any) to new array */ 3891 if (thresholds->primary) { 3892 memcpy(new->entries, thresholds->primary->entries, (size - 1) * 3893 sizeof(struct mem_cgroup_threshold)); 3894 } 3895 3896 /* Add new threshold */ 3897 new->entries[size - 1].eventfd = eventfd; 3898 new->entries[size - 1].threshold = threshold; 3899 3900 /* Sort thresholds. Registering of new threshold isn't time-critical */ 3901 sort(new->entries, size, sizeof(struct mem_cgroup_threshold), 3902 compare_thresholds, NULL); 3903 3904 /* Find current threshold */ 3905 new->current_threshold = -1; 3906 for (i = 0; i < size; i++) { 3907 if (new->entries[i].threshold < usage) { 3908 /* 3909 * new->current_threshold will not be used until 3910 * rcu_assign_pointer(), so it's safe to increment 3911 * it here. 3912 */ 3913 ++new->current_threshold; 3914 } 3915 } 3916 3917 /* Free old spare buffer and save old primary buffer as spare */ 3918 kfree(thresholds->spare); 3919 thresholds->spare = thresholds->primary; 3920 3921 rcu_assign_pointer(thresholds->primary, new); 3922 3923 /* To be sure that nobody uses thresholds */ 3924 synchronize_rcu(); 3925 3926 unlock: 3927 mutex_unlock(&memcg->thresholds_lock); 3928 3929 return ret; 3930 } 3931 3932 static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp, 3933 struct cftype *cft, struct eventfd_ctx *eventfd) 3934 { 3935 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); 3936 struct mem_cgroup_thresholds *thresholds; 3937 struct mem_cgroup_threshold_ary *new; 3938 int type = MEMFILE_TYPE(cft->private); 3939 u64 usage; 3940 int i, j, size; 3941 3942 mutex_lock(&memcg->thresholds_lock); 3943 if (type == _MEM) 3944 thresholds = &memcg->thresholds; 3945 else if (type == _MEMSWAP) 3946 thresholds = &memcg->memsw_thresholds; 3947 else 3948 BUG(); 3949 3950 /* 3951 * Something went wrong if we trying to unregister a threshold 3952 * if we don't have thresholds 3953 */ 3954 BUG_ON(!thresholds); 3955 3956 usage = mem_cgroup_usage(memcg, type == _MEMSWAP); 3957 3958 /* Check if a threshold crossed before removing */ 3959 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 3960 3961 /* Calculate new number of threshold */ 3962 size = 0; 3963 for (i = 0; i < thresholds->primary->size; i++) { 3964 if (thresholds->primary->entries[i].eventfd != eventfd) 3965 size++; 3966 } 3967 3968 new = thresholds->spare; 3969 3970 /* Set thresholds array to NULL if we don't have thresholds */ 3971 if (!size) { 3972 kfree(new); 3973 new = NULL; 3974 goto swap_buffers; 3975 } 3976 3977 new->size = size; 3978 3979 /* Copy thresholds and find current threshold */ 3980 new->current_threshold = -1; 3981 for (i = 0, j = 0; i < thresholds->primary->size; i++) { 3982 if (thresholds->primary->entries[i].eventfd == eventfd) 3983 continue; 3984 3985 new->entries[j] = thresholds->primary->entries[i]; 3986 if (new->entries[j].threshold < usage) { 3987 /* 3988 * new->current_threshold will not be used 3989 * until rcu_assign_pointer(), so it's safe to increment 3990 * it here. 3991 */ 3992 ++new->current_threshold; 3993 } 3994 j++; 3995 } 3996 3997 swap_buffers: 3998 /* Swap primary and spare array */ 3999 thresholds->spare = thresholds->primary; 4000 rcu_assign_pointer(thresholds->primary, new); 4001 4002 /* To be sure that nobody uses thresholds */ 4003 synchronize_rcu(); 4004 4005 mutex_unlock(&memcg->thresholds_lock); 4006 } 4007 4008 static int mem_cgroup_oom_register_event(struct cgroup *cgrp, 4009 struct cftype *cft, struct eventfd_ctx *eventfd, const char *args) 4010 { 4011 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); 4012 struct mem_cgroup_eventfd_list *event; 4013 int type = MEMFILE_TYPE(cft->private); 4014 4015 BUG_ON(type != _OOM_TYPE); 4016 event = kmalloc(sizeof(*event), GFP_KERNEL); 4017 if (!event) 4018 return -ENOMEM; 4019 4020 mutex_lock(&memcg_oom_mutex); 4021 4022 event->eventfd = eventfd; 4023 list_add(&event->list, &memcg->oom_notify); 4024 4025 /* already in OOM ? */ 4026 if (atomic_read(&memcg->oom_lock)) 4027 eventfd_signal(eventfd, 1); 4028 mutex_unlock(&memcg_oom_mutex); 4029 4030 return 0; 4031 } 4032 4033 static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp, 4034 struct cftype *cft, struct eventfd_ctx *eventfd) 4035 { 4036 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp); 4037 struct mem_cgroup_eventfd_list *ev, *tmp; 4038 int type = MEMFILE_TYPE(cft->private); 4039 4040 BUG_ON(type != _OOM_TYPE); 4041 4042 mutex_lock(&memcg_oom_mutex); 4043 4044 list_for_each_entry_safe(ev, tmp, &mem->oom_notify, list) { 4045 if (ev->eventfd == eventfd) { 4046 list_del(&ev->list); 4047 kfree(ev); 4048 } 4049 } 4050 4051 mutex_unlock(&memcg_oom_mutex); 4052 } 4053 4054 static int mem_cgroup_oom_control_read(struct cgroup *cgrp, 4055 struct cftype *cft, struct cgroup_map_cb *cb) 4056 { 4057 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp); 4058 4059 cb->fill(cb, "oom_kill_disable", mem->oom_kill_disable); 4060 4061 if (atomic_read(&mem->oom_lock)) 4062 cb->fill(cb, "under_oom", 1); 4063 else 4064 cb->fill(cb, "under_oom", 0); 4065 return 0; 4066 } 4067 4068 static int mem_cgroup_oom_control_write(struct cgroup *cgrp, 4069 struct cftype *cft, u64 val) 4070 { 4071 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp); 4072 struct mem_cgroup *parent; 4073 4074 /* cannot set to root cgroup and only 0 and 1 are allowed */ 4075 if (!cgrp->parent || !((val == 0) || (val == 1))) 4076 return -EINVAL; 4077 4078 parent = mem_cgroup_from_cont(cgrp->parent); 4079 4080 cgroup_lock(); 4081 /* oom-kill-disable is a flag for subhierarchy. */ 4082 if ((parent->use_hierarchy) || 4083 (mem->use_hierarchy && !list_empty(&cgrp->children))) { 4084 cgroup_unlock(); 4085 return -EINVAL; 4086 } 4087 mem->oom_kill_disable = val; 4088 if (!val) 4089 memcg_oom_recover(mem); 4090 cgroup_unlock(); 4091 return 0; 4092 } 4093 4094 static struct cftype mem_cgroup_files[] = { 4095 { 4096 .name = "usage_in_bytes", 4097 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE), 4098 .read_u64 = mem_cgroup_read, 4099 .register_event = mem_cgroup_usage_register_event, 4100 .unregister_event = mem_cgroup_usage_unregister_event, 4101 }, 4102 { 4103 .name = "max_usage_in_bytes", 4104 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE), 4105 .trigger = mem_cgroup_reset, 4106 .read_u64 = mem_cgroup_read, 4107 }, 4108 { 4109 .name = "limit_in_bytes", 4110 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT), 4111 .write_string = mem_cgroup_write, 4112 .read_u64 = mem_cgroup_read, 4113 }, 4114 { 4115 .name = "soft_limit_in_bytes", 4116 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT), 4117 .write_string = mem_cgroup_write, 4118 .read_u64 = mem_cgroup_read, 4119 }, 4120 { 4121 .name = "failcnt", 4122 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT), 4123 .trigger = mem_cgroup_reset, 4124 .read_u64 = mem_cgroup_read, 4125 }, 4126 { 4127 .name = "stat", 4128 .read_map = mem_control_stat_show, 4129 }, 4130 { 4131 .name = "force_empty", 4132 .trigger = mem_cgroup_force_empty_write, 4133 }, 4134 { 4135 .name = "use_hierarchy", 4136 .write_u64 = mem_cgroup_hierarchy_write, 4137 .read_u64 = mem_cgroup_hierarchy_read, 4138 }, 4139 { 4140 .name = "swappiness", 4141 .read_u64 = mem_cgroup_swappiness_read, 4142 .write_u64 = mem_cgroup_swappiness_write, 4143 }, 4144 { 4145 .name = "move_charge_at_immigrate", 4146 .read_u64 = mem_cgroup_move_charge_read, 4147 .write_u64 = mem_cgroup_move_charge_write, 4148 }, 4149 { 4150 .name = "oom_control", 4151 .read_map = mem_cgroup_oom_control_read, 4152 .write_u64 = mem_cgroup_oom_control_write, 4153 .register_event = mem_cgroup_oom_register_event, 4154 .unregister_event = mem_cgroup_oom_unregister_event, 4155 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL), 4156 }, 4157 }; 4158 4159 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP 4160 static struct cftype memsw_cgroup_files[] = { 4161 { 4162 .name = "memsw.usage_in_bytes", 4163 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE), 4164 .read_u64 = mem_cgroup_read, 4165 .register_event = mem_cgroup_usage_register_event, 4166 .unregister_event = mem_cgroup_usage_unregister_event, 4167 }, 4168 { 4169 .name = "memsw.max_usage_in_bytes", 4170 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE), 4171 .trigger = mem_cgroup_reset, 4172 .read_u64 = mem_cgroup_read, 4173 }, 4174 { 4175 .name = "memsw.limit_in_bytes", 4176 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT), 4177 .write_string = mem_cgroup_write, 4178 .read_u64 = mem_cgroup_read, 4179 }, 4180 { 4181 .name = "memsw.failcnt", 4182 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT), 4183 .trigger = mem_cgroup_reset, 4184 .read_u64 = mem_cgroup_read, 4185 }, 4186 }; 4187 4188 static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss) 4189 { 4190 if (!do_swap_account) 4191 return 0; 4192 return cgroup_add_files(cont, ss, memsw_cgroup_files, 4193 ARRAY_SIZE(memsw_cgroup_files)); 4194 }; 4195 #else 4196 static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss) 4197 { 4198 return 0; 4199 } 4200 #endif 4201 4202 static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node) 4203 { 4204 struct mem_cgroup_per_node *pn; 4205 struct mem_cgroup_per_zone *mz; 4206 enum lru_list l; 4207 int zone, tmp = node; 4208 /* 4209 * This routine is called against possible nodes. 4210 * But it's BUG to call kmalloc() against offline node. 4211 * 4212 * TODO: this routine can waste much memory for nodes which will 4213 * never be onlined. It's better to use memory hotplug callback 4214 * function. 4215 */ 4216 if (!node_state(node, N_NORMAL_MEMORY)) 4217 tmp = -1; 4218 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp); 4219 if (!pn) 4220 return 1; 4221 4222 mem->info.nodeinfo[node] = pn; 4223 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 4224 mz = &pn->zoneinfo[zone]; 4225 for_each_lru(l) 4226 INIT_LIST_HEAD(&mz->lists[l]); 4227 mz->usage_in_excess = 0; 4228 mz->on_tree = false; 4229 mz->mem = mem; 4230 } 4231 return 0; 4232 } 4233 4234 static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node) 4235 { 4236 kfree(mem->info.nodeinfo[node]); 4237 } 4238 4239 static struct mem_cgroup *mem_cgroup_alloc(void) 4240 { 4241 struct mem_cgroup *mem; 4242 int size = sizeof(struct mem_cgroup); 4243 4244 /* Can be very big if MAX_NUMNODES is very big */ 4245 if (size < PAGE_SIZE) 4246 mem = kzalloc(size, GFP_KERNEL); 4247 else 4248 mem = vzalloc(size); 4249 4250 if (!mem) 4251 return NULL; 4252 4253 mem->stat = alloc_percpu(struct mem_cgroup_stat_cpu); 4254 if (!mem->stat) 4255 goto out_free; 4256 spin_lock_init(&mem->pcp_counter_lock); 4257 return mem; 4258 4259 out_free: 4260 if (size < PAGE_SIZE) 4261 kfree(mem); 4262 else 4263 vfree(mem); 4264 return NULL; 4265 } 4266 4267 /* 4268 * At destroying mem_cgroup, references from swap_cgroup can remain. 4269 * (scanning all at force_empty is too costly...) 4270 * 4271 * Instead of clearing all references at force_empty, we remember 4272 * the number of reference from swap_cgroup and free mem_cgroup when 4273 * it goes down to 0. 4274 * 4275 * Removal of cgroup itself succeeds regardless of refs from swap. 4276 */ 4277 4278 static void __mem_cgroup_free(struct mem_cgroup *mem) 4279 { 4280 int node; 4281 4282 mem_cgroup_remove_from_trees(mem); 4283 free_css_id(&mem_cgroup_subsys, &mem->css); 4284 4285 for_each_node_state(node, N_POSSIBLE) 4286 free_mem_cgroup_per_zone_info(mem, node); 4287 4288 free_percpu(mem->stat); 4289 if (sizeof(struct mem_cgroup) < PAGE_SIZE) 4290 kfree(mem); 4291 else 4292 vfree(mem); 4293 } 4294 4295 static void mem_cgroup_get(struct mem_cgroup *mem) 4296 { 4297 atomic_inc(&mem->refcnt); 4298 } 4299 4300 static void __mem_cgroup_put(struct mem_cgroup *mem, int count) 4301 { 4302 if (atomic_sub_and_test(count, &mem->refcnt)) { 4303 struct mem_cgroup *parent = parent_mem_cgroup(mem); 4304 __mem_cgroup_free(mem); 4305 if (parent) 4306 mem_cgroup_put(parent); 4307 } 4308 } 4309 4310 static void mem_cgroup_put(struct mem_cgroup *mem) 4311 { 4312 __mem_cgroup_put(mem, 1); 4313 } 4314 4315 /* 4316 * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled. 4317 */ 4318 static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem) 4319 { 4320 if (!mem->res.parent) 4321 return NULL; 4322 return mem_cgroup_from_res_counter(mem->res.parent, res); 4323 } 4324 4325 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP 4326 static void __init enable_swap_cgroup(void) 4327 { 4328 if (!mem_cgroup_disabled() && really_do_swap_account) 4329 do_swap_account = 1; 4330 } 4331 #else 4332 static void __init enable_swap_cgroup(void) 4333 { 4334 } 4335 #endif 4336 4337 static int mem_cgroup_soft_limit_tree_init(void) 4338 { 4339 struct mem_cgroup_tree_per_node *rtpn; 4340 struct mem_cgroup_tree_per_zone *rtpz; 4341 int tmp, node, zone; 4342 4343 for_each_node_state(node, N_POSSIBLE) { 4344 tmp = node; 4345 if (!node_state(node, N_NORMAL_MEMORY)) 4346 tmp = -1; 4347 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp); 4348 if (!rtpn) 4349 return 1; 4350 4351 soft_limit_tree.rb_tree_per_node[node] = rtpn; 4352 4353 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 4354 rtpz = &rtpn->rb_tree_per_zone[zone]; 4355 rtpz->rb_root = RB_ROOT; 4356 spin_lock_init(&rtpz->lock); 4357 } 4358 } 4359 return 0; 4360 } 4361 4362 static struct cgroup_subsys_state * __ref 4363 mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont) 4364 { 4365 struct mem_cgroup *mem, *parent; 4366 long error = -ENOMEM; 4367 int node; 4368 4369 mem = mem_cgroup_alloc(); 4370 if (!mem) 4371 return ERR_PTR(error); 4372 4373 for_each_node_state(node, N_POSSIBLE) 4374 if (alloc_mem_cgroup_per_zone_info(mem, node)) 4375 goto free_out; 4376 4377 /* root ? */ 4378 if (cont->parent == NULL) { 4379 int cpu; 4380 enable_swap_cgroup(); 4381 parent = NULL; 4382 root_mem_cgroup = mem; 4383 if (mem_cgroup_soft_limit_tree_init()) 4384 goto free_out; 4385 for_each_possible_cpu(cpu) { 4386 struct memcg_stock_pcp *stock = 4387 &per_cpu(memcg_stock, cpu); 4388 INIT_WORK(&stock->work, drain_local_stock); 4389 } 4390 hotcpu_notifier(memcg_cpu_hotplug_callback, 0); 4391 } else { 4392 parent = mem_cgroup_from_cont(cont->parent); 4393 mem->use_hierarchy = parent->use_hierarchy; 4394 mem->oom_kill_disable = parent->oom_kill_disable; 4395 } 4396 4397 if (parent && parent->use_hierarchy) { 4398 res_counter_init(&mem->res, &parent->res); 4399 res_counter_init(&mem->memsw, &parent->memsw); 4400 /* 4401 * We increment refcnt of the parent to ensure that we can 4402 * safely access it on res_counter_charge/uncharge. 4403 * This refcnt will be decremented when freeing this 4404 * mem_cgroup(see mem_cgroup_put). 4405 */ 4406 mem_cgroup_get(parent); 4407 } else { 4408 res_counter_init(&mem->res, NULL); 4409 res_counter_init(&mem->memsw, NULL); 4410 } 4411 mem->last_scanned_child = 0; 4412 spin_lock_init(&mem->reclaim_param_lock); 4413 INIT_LIST_HEAD(&mem->oom_notify); 4414 4415 if (parent) 4416 mem->swappiness = get_swappiness(parent); 4417 atomic_set(&mem->refcnt, 1); 4418 mem->move_charge_at_immigrate = 0; 4419 mutex_init(&mem->thresholds_lock); 4420 return &mem->css; 4421 free_out: 4422 __mem_cgroup_free(mem); 4423 root_mem_cgroup = NULL; 4424 return ERR_PTR(error); 4425 } 4426 4427 static int mem_cgroup_pre_destroy(struct cgroup_subsys *ss, 4428 struct cgroup *cont) 4429 { 4430 struct mem_cgroup *mem = mem_cgroup_from_cont(cont); 4431 4432 return mem_cgroup_force_empty(mem, false); 4433 } 4434 4435 static void mem_cgroup_destroy(struct cgroup_subsys *ss, 4436 struct cgroup *cont) 4437 { 4438 struct mem_cgroup *mem = mem_cgroup_from_cont(cont); 4439 4440 mem_cgroup_put(mem); 4441 } 4442 4443 static int mem_cgroup_populate(struct cgroup_subsys *ss, 4444 struct cgroup *cont) 4445 { 4446 int ret; 4447 4448 ret = cgroup_add_files(cont, ss, mem_cgroup_files, 4449 ARRAY_SIZE(mem_cgroup_files)); 4450 4451 if (!ret) 4452 ret = register_memsw_files(cont, ss); 4453 return ret; 4454 } 4455 4456 #ifdef CONFIG_MMU 4457 /* Handlers for move charge at task migration. */ 4458 #define PRECHARGE_COUNT_AT_ONCE 256 4459 static int mem_cgroup_do_precharge(unsigned long count) 4460 { 4461 int ret = 0; 4462 int batch_count = PRECHARGE_COUNT_AT_ONCE; 4463 struct mem_cgroup *mem = mc.to; 4464 4465 if (mem_cgroup_is_root(mem)) { 4466 mc.precharge += count; 4467 /* we don't need css_get for root */ 4468 return ret; 4469 } 4470 /* try to charge at once */ 4471 if (count > 1) { 4472 struct res_counter *dummy; 4473 /* 4474 * "mem" cannot be under rmdir() because we've already checked 4475 * by cgroup_lock_live_cgroup() that it is not removed and we 4476 * are still under the same cgroup_mutex. So we can postpone 4477 * css_get(). 4478 */ 4479 if (res_counter_charge(&mem->res, PAGE_SIZE * count, &dummy)) 4480 goto one_by_one; 4481 if (do_swap_account && res_counter_charge(&mem->memsw, 4482 PAGE_SIZE * count, &dummy)) { 4483 res_counter_uncharge(&mem->res, PAGE_SIZE * count); 4484 goto one_by_one; 4485 } 4486 mc.precharge += count; 4487 return ret; 4488 } 4489 one_by_one: 4490 /* fall back to one by one charge */ 4491 while (count--) { 4492 if (signal_pending(current)) { 4493 ret = -EINTR; 4494 break; 4495 } 4496 if (!batch_count--) { 4497 batch_count = PRECHARGE_COUNT_AT_ONCE; 4498 cond_resched(); 4499 } 4500 ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem, false, 4501 PAGE_SIZE); 4502 if (ret || !mem) 4503 /* mem_cgroup_clear_mc() will do uncharge later */ 4504 return -ENOMEM; 4505 mc.precharge++; 4506 } 4507 return ret; 4508 } 4509 4510 /** 4511 * is_target_pte_for_mc - check a pte whether it is valid for move charge 4512 * @vma: the vma the pte to be checked belongs 4513 * @addr: the address corresponding to the pte to be checked 4514 * @ptent: the pte to be checked 4515 * @target: the pointer the target page or swap ent will be stored(can be NULL) 4516 * 4517 * Returns 4518 * 0(MC_TARGET_NONE): if the pte is not a target for move charge. 4519 * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for 4520 * move charge. if @target is not NULL, the page is stored in target->page 4521 * with extra refcnt got(Callers should handle it). 4522 * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a 4523 * target for charge migration. if @target is not NULL, the entry is stored 4524 * in target->ent. 4525 * 4526 * Called with pte lock held. 4527 */ 4528 union mc_target { 4529 struct page *page; 4530 swp_entry_t ent; 4531 }; 4532 4533 enum mc_target_type { 4534 MC_TARGET_NONE, /* not used */ 4535 MC_TARGET_PAGE, 4536 MC_TARGET_SWAP, 4537 }; 4538 4539 static struct page *mc_handle_present_pte(struct vm_area_struct *vma, 4540 unsigned long addr, pte_t ptent) 4541 { 4542 struct page *page = vm_normal_page(vma, addr, ptent); 4543 4544 if (!page || !page_mapped(page)) 4545 return NULL; 4546 if (PageAnon(page)) { 4547 /* we don't move shared anon */ 4548 if (!move_anon() || page_mapcount(page) > 2) 4549 return NULL; 4550 } else if (!move_file()) 4551 /* we ignore mapcount for file pages */ 4552 return NULL; 4553 if (!get_page_unless_zero(page)) 4554 return NULL; 4555 4556 return page; 4557 } 4558 4559 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma, 4560 unsigned long addr, pte_t ptent, swp_entry_t *entry) 4561 { 4562 int usage_count; 4563 struct page *page = NULL; 4564 swp_entry_t ent = pte_to_swp_entry(ptent); 4565 4566 if (!move_anon() || non_swap_entry(ent)) 4567 return NULL; 4568 usage_count = mem_cgroup_count_swap_user(ent, &page); 4569 if (usage_count > 1) { /* we don't move shared anon */ 4570 if (page) 4571 put_page(page); 4572 return NULL; 4573 } 4574 if (do_swap_account) 4575 entry->val = ent.val; 4576 4577 return page; 4578 } 4579 4580 static struct page *mc_handle_file_pte(struct vm_area_struct *vma, 4581 unsigned long addr, pte_t ptent, swp_entry_t *entry) 4582 { 4583 struct page *page = NULL; 4584 struct inode *inode; 4585 struct address_space *mapping; 4586 pgoff_t pgoff; 4587 4588 if (!vma->vm_file) /* anonymous vma */ 4589 return NULL; 4590 if (!move_file()) 4591 return NULL; 4592 4593 inode = vma->vm_file->f_path.dentry->d_inode; 4594 mapping = vma->vm_file->f_mapping; 4595 if (pte_none(ptent)) 4596 pgoff = linear_page_index(vma, addr); 4597 else /* pte_file(ptent) is true */ 4598 pgoff = pte_to_pgoff(ptent); 4599 4600 /* page is moved even if it's not RSS of this task(page-faulted). */ 4601 if (!mapping_cap_swap_backed(mapping)) { /* normal file */ 4602 page = find_get_page(mapping, pgoff); 4603 } else { /* shmem/tmpfs file. we should take account of swap too. */ 4604 swp_entry_t ent; 4605 mem_cgroup_get_shmem_target(inode, pgoff, &page, &ent); 4606 if (do_swap_account) 4607 entry->val = ent.val; 4608 } 4609 4610 return page; 4611 } 4612 4613 static int is_target_pte_for_mc(struct vm_area_struct *vma, 4614 unsigned long addr, pte_t ptent, union mc_target *target) 4615 { 4616 struct page *page = NULL; 4617 struct page_cgroup *pc; 4618 int ret = 0; 4619 swp_entry_t ent = { .val = 0 }; 4620 4621 if (pte_present(ptent)) 4622 page = mc_handle_present_pte(vma, addr, ptent); 4623 else if (is_swap_pte(ptent)) 4624 page = mc_handle_swap_pte(vma, addr, ptent, &ent); 4625 else if (pte_none(ptent) || pte_file(ptent)) 4626 page = mc_handle_file_pte(vma, addr, ptent, &ent); 4627 4628 if (!page && !ent.val) 4629 return 0; 4630 if (page) { 4631 pc = lookup_page_cgroup(page); 4632 /* 4633 * Do only loose check w/o page_cgroup lock. 4634 * mem_cgroup_move_account() checks the pc is valid or not under 4635 * the lock. 4636 */ 4637 if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) { 4638 ret = MC_TARGET_PAGE; 4639 if (target) 4640 target->page = page; 4641 } 4642 if (!ret || !target) 4643 put_page(page); 4644 } 4645 /* There is a swap entry and a page doesn't exist or isn't charged */ 4646 if (ent.val && !ret && 4647 css_id(&mc.from->css) == lookup_swap_cgroup(ent)) { 4648 ret = MC_TARGET_SWAP; 4649 if (target) 4650 target->ent = ent; 4651 } 4652 return ret; 4653 } 4654 4655 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd, 4656 unsigned long addr, unsigned long end, 4657 struct mm_walk *walk) 4658 { 4659 struct vm_area_struct *vma = walk->private; 4660 pte_t *pte; 4661 spinlock_t *ptl; 4662 4663 VM_BUG_ON(pmd_trans_huge(*pmd)); 4664 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 4665 for (; addr != end; pte++, addr += PAGE_SIZE) 4666 if (is_target_pte_for_mc(vma, addr, *pte, NULL)) 4667 mc.precharge++; /* increment precharge temporarily */ 4668 pte_unmap_unlock(pte - 1, ptl); 4669 cond_resched(); 4670 4671 return 0; 4672 } 4673 4674 static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm) 4675 { 4676 unsigned long precharge; 4677 struct vm_area_struct *vma; 4678 4679 down_read(&mm->mmap_sem); 4680 for (vma = mm->mmap; vma; vma = vma->vm_next) { 4681 struct mm_walk mem_cgroup_count_precharge_walk = { 4682 .pmd_entry = mem_cgroup_count_precharge_pte_range, 4683 .mm = mm, 4684 .private = vma, 4685 }; 4686 if (is_vm_hugetlb_page(vma)) 4687 continue; 4688 walk_page_range(vma->vm_start, vma->vm_end, 4689 &mem_cgroup_count_precharge_walk); 4690 } 4691 up_read(&mm->mmap_sem); 4692 4693 precharge = mc.precharge; 4694 mc.precharge = 0; 4695 4696 return precharge; 4697 } 4698 4699 static int mem_cgroup_precharge_mc(struct mm_struct *mm) 4700 { 4701 unsigned long precharge = mem_cgroup_count_precharge(mm); 4702 4703 VM_BUG_ON(mc.moving_task); 4704 mc.moving_task = current; 4705 return mem_cgroup_do_precharge(precharge); 4706 } 4707 4708 /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */ 4709 static void __mem_cgroup_clear_mc(void) 4710 { 4711 struct mem_cgroup *from = mc.from; 4712 struct mem_cgroup *to = mc.to; 4713 4714 /* we must uncharge all the leftover precharges from mc.to */ 4715 if (mc.precharge) { 4716 __mem_cgroup_cancel_charge(mc.to, mc.precharge); 4717 mc.precharge = 0; 4718 } 4719 /* 4720 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so 4721 * we must uncharge here. 4722 */ 4723 if (mc.moved_charge) { 4724 __mem_cgroup_cancel_charge(mc.from, mc.moved_charge); 4725 mc.moved_charge = 0; 4726 } 4727 /* we must fixup refcnts and charges */ 4728 if (mc.moved_swap) { 4729 /* uncharge swap account from the old cgroup */ 4730 if (!mem_cgroup_is_root(mc.from)) 4731 res_counter_uncharge(&mc.from->memsw, 4732 PAGE_SIZE * mc.moved_swap); 4733 __mem_cgroup_put(mc.from, mc.moved_swap); 4734 4735 if (!mem_cgroup_is_root(mc.to)) { 4736 /* 4737 * we charged both to->res and to->memsw, so we should 4738 * uncharge to->res. 4739 */ 4740 res_counter_uncharge(&mc.to->res, 4741 PAGE_SIZE * mc.moved_swap); 4742 } 4743 /* we've already done mem_cgroup_get(mc.to) */ 4744 mc.moved_swap = 0; 4745 } 4746 memcg_oom_recover(from); 4747 memcg_oom_recover(to); 4748 wake_up_all(&mc.waitq); 4749 } 4750 4751 static void mem_cgroup_clear_mc(void) 4752 { 4753 struct mem_cgroup *from = mc.from; 4754 4755 /* 4756 * we must clear moving_task before waking up waiters at the end of 4757 * task migration. 4758 */ 4759 mc.moving_task = NULL; 4760 __mem_cgroup_clear_mc(); 4761 spin_lock(&mc.lock); 4762 mc.from = NULL; 4763 mc.to = NULL; 4764 spin_unlock(&mc.lock); 4765 mem_cgroup_end_move(from); 4766 } 4767 4768 static int mem_cgroup_can_attach(struct cgroup_subsys *ss, 4769 struct cgroup *cgroup, 4770 struct task_struct *p, 4771 bool threadgroup) 4772 { 4773 int ret = 0; 4774 struct mem_cgroup *mem = mem_cgroup_from_cont(cgroup); 4775 4776 if (mem->move_charge_at_immigrate) { 4777 struct mm_struct *mm; 4778 struct mem_cgroup *from = mem_cgroup_from_task(p); 4779 4780 VM_BUG_ON(from == mem); 4781 4782 mm = get_task_mm(p); 4783 if (!mm) 4784 return 0; 4785 /* We move charges only when we move a owner of the mm */ 4786 if (mm->owner == p) { 4787 VM_BUG_ON(mc.from); 4788 VM_BUG_ON(mc.to); 4789 VM_BUG_ON(mc.precharge); 4790 VM_BUG_ON(mc.moved_charge); 4791 VM_BUG_ON(mc.moved_swap); 4792 mem_cgroup_start_move(from); 4793 spin_lock(&mc.lock); 4794 mc.from = from; 4795 mc.to = mem; 4796 spin_unlock(&mc.lock); 4797 /* We set mc.moving_task later */ 4798 4799 ret = mem_cgroup_precharge_mc(mm); 4800 if (ret) 4801 mem_cgroup_clear_mc(); 4802 } 4803 mmput(mm); 4804 } 4805 return ret; 4806 } 4807 4808 static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss, 4809 struct cgroup *cgroup, 4810 struct task_struct *p, 4811 bool threadgroup) 4812 { 4813 mem_cgroup_clear_mc(); 4814 } 4815 4816 static int mem_cgroup_move_charge_pte_range(pmd_t *pmd, 4817 unsigned long addr, unsigned long end, 4818 struct mm_walk *walk) 4819 { 4820 int ret = 0; 4821 struct vm_area_struct *vma = walk->private; 4822 pte_t *pte; 4823 spinlock_t *ptl; 4824 4825 retry: 4826 VM_BUG_ON(pmd_trans_huge(*pmd)); 4827 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 4828 for (; addr != end; addr += PAGE_SIZE) { 4829 pte_t ptent = *(pte++); 4830 union mc_target target; 4831 int type; 4832 struct page *page; 4833 struct page_cgroup *pc; 4834 swp_entry_t ent; 4835 4836 if (!mc.precharge) 4837 break; 4838 4839 type = is_target_pte_for_mc(vma, addr, ptent, &target); 4840 switch (type) { 4841 case MC_TARGET_PAGE: 4842 page = target.page; 4843 if (isolate_lru_page(page)) 4844 goto put; 4845 pc = lookup_page_cgroup(page); 4846 if (!mem_cgroup_move_account(pc, 4847 mc.from, mc.to, false)) { 4848 mc.precharge--; 4849 /* we uncharge from mc.from later. */ 4850 mc.moved_charge++; 4851 } 4852 putback_lru_page(page); 4853 put: /* is_target_pte_for_mc() gets the page */ 4854 put_page(page); 4855 break; 4856 case MC_TARGET_SWAP: 4857 ent = target.ent; 4858 if (!mem_cgroup_move_swap_account(ent, 4859 mc.from, mc.to, false)) { 4860 mc.precharge--; 4861 /* we fixup refcnts and charges later. */ 4862 mc.moved_swap++; 4863 } 4864 break; 4865 default: 4866 break; 4867 } 4868 } 4869 pte_unmap_unlock(pte - 1, ptl); 4870 cond_resched(); 4871 4872 if (addr != end) { 4873 /* 4874 * We have consumed all precharges we got in can_attach(). 4875 * We try charge one by one, but don't do any additional 4876 * charges to mc.to if we have failed in charge once in attach() 4877 * phase. 4878 */ 4879 ret = mem_cgroup_do_precharge(1); 4880 if (!ret) 4881 goto retry; 4882 } 4883 4884 return ret; 4885 } 4886 4887 static void mem_cgroup_move_charge(struct mm_struct *mm) 4888 { 4889 struct vm_area_struct *vma; 4890 4891 lru_add_drain_all(); 4892 retry: 4893 if (unlikely(!down_read_trylock(&mm->mmap_sem))) { 4894 /* 4895 * Someone who are holding the mmap_sem might be waiting in 4896 * waitq. So we cancel all extra charges, wake up all waiters, 4897 * and retry. Because we cancel precharges, we might not be able 4898 * to move enough charges, but moving charge is a best-effort 4899 * feature anyway, so it wouldn't be a big problem. 4900 */ 4901 __mem_cgroup_clear_mc(); 4902 cond_resched(); 4903 goto retry; 4904 } 4905 for (vma = mm->mmap; vma; vma = vma->vm_next) { 4906 int ret; 4907 struct mm_walk mem_cgroup_move_charge_walk = { 4908 .pmd_entry = mem_cgroup_move_charge_pte_range, 4909 .mm = mm, 4910 .private = vma, 4911 }; 4912 if (is_vm_hugetlb_page(vma)) 4913 continue; 4914 ret = walk_page_range(vma->vm_start, vma->vm_end, 4915 &mem_cgroup_move_charge_walk); 4916 if (ret) 4917 /* 4918 * means we have consumed all precharges and failed in 4919 * doing additional charge. Just abandon here. 4920 */ 4921 break; 4922 } 4923 up_read(&mm->mmap_sem); 4924 } 4925 4926 static void mem_cgroup_move_task(struct cgroup_subsys *ss, 4927 struct cgroup *cont, 4928 struct cgroup *old_cont, 4929 struct task_struct *p, 4930 bool threadgroup) 4931 { 4932 struct mm_struct *mm; 4933 4934 if (!mc.to) 4935 /* no need to move charge */ 4936 return; 4937 4938 mm = get_task_mm(p); 4939 if (mm) { 4940 mem_cgroup_move_charge(mm); 4941 mmput(mm); 4942 } 4943 mem_cgroup_clear_mc(); 4944 } 4945 #else /* !CONFIG_MMU */ 4946 static int mem_cgroup_can_attach(struct cgroup_subsys *ss, 4947 struct cgroup *cgroup, 4948 struct task_struct *p, 4949 bool threadgroup) 4950 { 4951 return 0; 4952 } 4953 static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss, 4954 struct cgroup *cgroup, 4955 struct task_struct *p, 4956 bool threadgroup) 4957 { 4958 } 4959 static void mem_cgroup_move_task(struct cgroup_subsys *ss, 4960 struct cgroup *cont, 4961 struct cgroup *old_cont, 4962 struct task_struct *p, 4963 bool threadgroup) 4964 { 4965 } 4966 #endif 4967 4968 struct cgroup_subsys mem_cgroup_subsys = { 4969 .name = "memory", 4970 .subsys_id = mem_cgroup_subsys_id, 4971 .create = mem_cgroup_create, 4972 .pre_destroy = mem_cgroup_pre_destroy, 4973 .destroy = mem_cgroup_destroy, 4974 .populate = mem_cgroup_populate, 4975 .can_attach = mem_cgroup_can_attach, 4976 .cancel_attach = mem_cgroup_cancel_attach, 4977 .attach = mem_cgroup_move_task, 4978 .early_init = 0, 4979 .use_id = 1, 4980 }; 4981 4982 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP 4983 static int __init enable_swap_account(char *s) 4984 { 4985 /* consider enabled if no parameter or 1 is given */ 4986 if (!s || !strcmp(s, "1")) 4987 really_do_swap_account = 1; 4988 else if (!strcmp(s, "0")) 4989 really_do_swap_account = 0; 4990 return 1; 4991 } 4992 __setup("swapaccount", enable_swap_account); 4993 4994 static int __init disable_swap_account(char *s) 4995 { 4996 enable_swap_account("0"); 4997 return 1; 4998 } 4999 __setup("noswapaccount", disable_swap_account); 5000 #endif 5001