1 /* memcontrol.c - Memory Controller 2 * 3 * Copyright IBM Corporation, 2007 4 * Author Balbir Singh <balbir@linux.vnet.ibm.com> 5 * 6 * Copyright 2007 OpenVZ SWsoft Inc 7 * Author: Pavel Emelianov <xemul@openvz.org> 8 * 9 * Memory thresholds 10 * Copyright (C) 2009 Nokia Corporation 11 * Author: Kirill A. Shutemov 12 * 13 * Kernel Memory Controller 14 * Copyright (C) 2012 Parallels Inc. and Google Inc. 15 * Authors: Glauber Costa and Suleiman Souhlal 16 * 17 * Native page reclaim 18 * Charge lifetime sanitation 19 * Lockless page tracking & accounting 20 * Unified hierarchy configuration model 21 * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner 22 * 23 * This program is free software; you can redistribute it and/or modify 24 * it under the terms of the GNU General Public License as published by 25 * the Free Software Foundation; either version 2 of the License, or 26 * (at your option) any later version. 27 * 28 * This program is distributed in the hope that it will be useful, 29 * but WITHOUT ANY WARRANTY; without even the implied warranty of 30 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 * GNU General Public License for more details. 32 */ 33 34 #include <linux/page_counter.h> 35 #include <linux/memcontrol.h> 36 #include <linux/cgroup.h> 37 #include <linux/mm.h> 38 #include <linux/hugetlb.h> 39 #include <linux/pagemap.h> 40 #include <linux/smp.h> 41 #include <linux/page-flags.h> 42 #include <linux/backing-dev.h> 43 #include <linux/bit_spinlock.h> 44 #include <linux/rcupdate.h> 45 #include <linux/limits.h> 46 #include <linux/export.h> 47 #include <linux/mutex.h> 48 #include <linux/rbtree.h> 49 #include <linux/slab.h> 50 #include <linux/swap.h> 51 #include <linux/swapops.h> 52 #include <linux/spinlock.h> 53 #include <linux/eventfd.h> 54 #include <linux/poll.h> 55 #include <linux/sort.h> 56 #include <linux/fs.h> 57 #include <linux/seq_file.h> 58 #include <linux/vmpressure.h> 59 #include <linux/mm_inline.h> 60 #include <linux/swap_cgroup.h> 61 #include <linux/cpu.h> 62 #include <linux/oom.h> 63 #include <linux/lockdep.h> 64 #include <linux/file.h> 65 #include <linux/tracehook.h> 66 #include "internal.h" 67 #include <net/sock.h> 68 #include <net/ip.h> 69 #include "slab.h" 70 71 #include <asm/uaccess.h> 72 73 #include <trace/events/vmscan.h> 74 75 struct cgroup_subsys memory_cgrp_subsys __read_mostly; 76 EXPORT_SYMBOL(memory_cgrp_subsys); 77 78 struct mem_cgroup *root_mem_cgroup __read_mostly; 79 80 #define MEM_CGROUP_RECLAIM_RETRIES 5 81 82 /* Socket memory accounting disabled? */ 83 static bool cgroup_memory_nosocket; 84 85 /* Kernel memory accounting disabled? */ 86 static bool cgroup_memory_nokmem; 87 88 /* Whether the swap controller is active */ 89 #ifdef CONFIG_MEMCG_SWAP 90 int do_swap_account __read_mostly; 91 #else 92 #define do_swap_account 0 93 #endif 94 95 /* Whether legacy memory+swap accounting is active */ 96 static bool do_memsw_account(void) 97 { 98 return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && do_swap_account; 99 } 100 101 static const char * const mem_cgroup_stat_names[] = { 102 "cache", 103 "rss", 104 "rss_huge", 105 "mapped_file", 106 "dirty", 107 "writeback", 108 "swap", 109 }; 110 111 static const char * const mem_cgroup_events_names[] = { 112 "pgpgin", 113 "pgpgout", 114 "pgfault", 115 "pgmajfault", 116 }; 117 118 static const char * const mem_cgroup_lru_names[] = { 119 "inactive_anon", 120 "active_anon", 121 "inactive_file", 122 "active_file", 123 "unevictable", 124 }; 125 126 #define THRESHOLDS_EVENTS_TARGET 128 127 #define SOFTLIMIT_EVENTS_TARGET 1024 128 #define NUMAINFO_EVENTS_TARGET 1024 129 130 /* 131 * Cgroups above their limits are maintained in a RB-Tree, independent of 132 * their hierarchy representation 133 */ 134 135 struct mem_cgroup_tree_per_zone { 136 struct rb_root rb_root; 137 spinlock_t lock; 138 }; 139 140 struct mem_cgroup_tree_per_node { 141 struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES]; 142 }; 143 144 struct mem_cgroup_tree { 145 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES]; 146 }; 147 148 static struct mem_cgroup_tree soft_limit_tree __read_mostly; 149 150 /* for OOM */ 151 struct mem_cgroup_eventfd_list { 152 struct list_head list; 153 struct eventfd_ctx *eventfd; 154 }; 155 156 /* 157 * cgroup_event represents events which userspace want to receive. 158 */ 159 struct mem_cgroup_event { 160 /* 161 * memcg which the event belongs to. 162 */ 163 struct mem_cgroup *memcg; 164 /* 165 * eventfd to signal userspace about the event. 166 */ 167 struct eventfd_ctx *eventfd; 168 /* 169 * Each of these stored in a list by the cgroup. 170 */ 171 struct list_head list; 172 /* 173 * register_event() callback will be used to add new userspace 174 * waiter for changes related to this event. Use eventfd_signal() 175 * on eventfd to send notification to userspace. 176 */ 177 int (*register_event)(struct mem_cgroup *memcg, 178 struct eventfd_ctx *eventfd, const char *args); 179 /* 180 * unregister_event() callback will be called when userspace closes 181 * the eventfd or on cgroup removing. This callback must be set, 182 * if you want provide notification functionality. 183 */ 184 void (*unregister_event)(struct mem_cgroup *memcg, 185 struct eventfd_ctx *eventfd); 186 /* 187 * All fields below needed to unregister event when 188 * userspace closes eventfd. 189 */ 190 poll_table pt; 191 wait_queue_head_t *wqh; 192 wait_queue_t wait; 193 struct work_struct remove; 194 }; 195 196 static void mem_cgroup_threshold(struct mem_cgroup *memcg); 197 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg); 198 199 /* Stuffs for move charges at task migration. */ 200 /* 201 * Types of charges to be moved. 202 */ 203 #define MOVE_ANON 0x1U 204 #define MOVE_FILE 0x2U 205 #define MOVE_MASK (MOVE_ANON | MOVE_FILE) 206 207 /* "mc" and its members are protected by cgroup_mutex */ 208 static struct move_charge_struct { 209 spinlock_t lock; /* for from, to */ 210 struct mm_struct *mm; 211 struct mem_cgroup *from; 212 struct mem_cgroup *to; 213 unsigned long flags; 214 unsigned long precharge; 215 unsigned long moved_charge; 216 unsigned long moved_swap; 217 struct task_struct *moving_task; /* a task moving charges */ 218 wait_queue_head_t waitq; /* a waitq for other context */ 219 } mc = { 220 .lock = __SPIN_LOCK_UNLOCKED(mc.lock), 221 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq), 222 }; 223 224 /* 225 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft 226 * limit reclaim to prevent infinite loops, if they ever occur. 227 */ 228 #define MEM_CGROUP_MAX_RECLAIM_LOOPS 100 229 #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2 230 231 enum charge_type { 232 MEM_CGROUP_CHARGE_TYPE_CACHE = 0, 233 MEM_CGROUP_CHARGE_TYPE_ANON, 234 MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */ 235 MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */ 236 NR_CHARGE_TYPE, 237 }; 238 239 /* for encoding cft->private value on file */ 240 enum res_type { 241 _MEM, 242 _MEMSWAP, 243 _OOM_TYPE, 244 _KMEM, 245 _TCP, 246 }; 247 248 #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val)) 249 #define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff) 250 #define MEMFILE_ATTR(val) ((val) & 0xffff) 251 /* Used for OOM nofiier */ 252 #define OOM_CONTROL (0) 253 254 /* Some nice accessors for the vmpressure. */ 255 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg) 256 { 257 if (!memcg) 258 memcg = root_mem_cgroup; 259 return &memcg->vmpressure; 260 } 261 262 struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr) 263 { 264 return &container_of(vmpr, struct mem_cgroup, vmpressure)->css; 265 } 266 267 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg) 268 { 269 return (memcg == root_mem_cgroup); 270 } 271 272 #ifndef CONFIG_SLOB 273 /* 274 * This will be the memcg's index in each cache's ->memcg_params.memcg_caches. 275 * The main reason for not using cgroup id for this: 276 * this works better in sparse environments, where we have a lot of memcgs, 277 * but only a few kmem-limited. Or also, if we have, for instance, 200 278 * memcgs, and none but the 200th is kmem-limited, we'd have to have a 279 * 200 entry array for that. 280 * 281 * The current size of the caches array is stored in memcg_nr_cache_ids. It 282 * will double each time we have to increase it. 283 */ 284 static DEFINE_IDA(memcg_cache_ida); 285 int memcg_nr_cache_ids; 286 287 /* Protects memcg_nr_cache_ids */ 288 static DECLARE_RWSEM(memcg_cache_ids_sem); 289 290 void memcg_get_cache_ids(void) 291 { 292 down_read(&memcg_cache_ids_sem); 293 } 294 295 void memcg_put_cache_ids(void) 296 { 297 up_read(&memcg_cache_ids_sem); 298 } 299 300 /* 301 * MIN_SIZE is different than 1, because we would like to avoid going through 302 * the alloc/free process all the time. In a small machine, 4 kmem-limited 303 * cgroups is a reasonable guess. In the future, it could be a parameter or 304 * tunable, but that is strictly not necessary. 305 * 306 * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get 307 * this constant directly from cgroup, but it is understandable that this is 308 * better kept as an internal representation in cgroup.c. In any case, the 309 * cgrp_id space is not getting any smaller, and we don't have to necessarily 310 * increase ours as well if it increases. 311 */ 312 #define MEMCG_CACHES_MIN_SIZE 4 313 #define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX 314 315 /* 316 * A lot of the calls to the cache allocation functions are expected to be 317 * inlined by the compiler. Since the calls to memcg_kmem_get_cache are 318 * conditional to this static branch, we'll have to allow modules that does 319 * kmem_cache_alloc and the such to see this symbol as well 320 */ 321 DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key); 322 EXPORT_SYMBOL(memcg_kmem_enabled_key); 323 324 #endif /* !CONFIG_SLOB */ 325 326 static struct mem_cgroup_per_zone * 327 mem_cgroup_zone_zoneinfo(struct mem_cgroup *memcg, struct zone *zone) 328 { 329 int nid = zone_to_nid(zone); 330 int zid = zone_idx(zone); 331 332 return &memcg->nodeinfo[nid]->zoneinfo[zid]; 333 } 334 335 /** 336 * mem_cgroup_css_from_page - css of the memcg associated with a page 337 * @page: page of interest 338 * 339 * If memcg is bound to the default hierarchy, css of the memcg associated 340 * with @page is returned. The returned css remains associated with @page 341 * until it is released. 342 * 343 * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup 344 * is returned. 345 */ 346 struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page) 347 { 348 struct mem_cgroup *memcg; 349 350 memcg = page->mem_cgroup; 351 352 if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys)) 353 memcg = root_mem_cgroup; 354 355 return &memcg->css; 356 } 357 358 /** 359 * page_cgroup_ino - return inode number of the memcg a page is charged to 360 * @page: the page 361 * 362 * Look up the closest online ancestor of the memory cgroup @page is charged to 363 * and return its inode number or 0 if @page is not charged to any cgroup. It 364 * is safe to call this function without holding a reference to @page. 365 * 366 * Note, this function is inherently racy, because there is nothing to prevent 367 * the cgroup inode from getting torn down and potentially reallocated a moment 368 * after page_cgroup_ino() returns, so it only should be used by callers that 369 * do not care (such as procfs interfaces). 370 */ 371 ino_t page_cgroup_ino(struct page *page) 372 { 373 struct mem_cgroup *memcg; 374 unsigned long ino = 0; 375 376 rcu_read_lock(); 377 memcg = READ_ONCE(page->mem_cgroup); 378 while (memcg && !(memcg->css.flags & CSS_ONLINE)) 379 memcg = parent_mem_cgroup(memcg); 380 if (memcg) 381 ino = cgroup_ino(memcg->css.cgroup); 382 rcu_read_unlock(); 383 return ino; 384 } 385 386 static struct mem_cgroup_per_zone * 387 mem_cgroup_page_zoneinfo(struct mem_cgroup *memcg, struct page *page) 388 { 389 int nid = page_to_nid(page); 390 int zid = page_zonenum(page); 391 392 return &memcg->nodeinfo[nid]->zoneinfo[zid]; 393 } 394 395 static struct mem_cgroup_tree_per_zone * 396 soft_limit_tree_node_zone(int nid, int zid) 397 { 398 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid]; 399 } 400 401 static struct mem_cgroup_tree_per_zone * 402 soft_limit_tree_from_page(struct page *page) 403 { 404 int nid = page_to_nid(page); 405 int zid = page_zonenum(page); 406 407 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid]; 408 } 409 410 static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_zone *mz, 411 struct mem_cgroup_tree_per_zone *mctz, 412 unsigned long new_usage_in_excess) 413 { 414 struct rb_node **p = &mctz->rb_root.rb_node; 415 struct rb_node *parent = NULL; 416 struct mem_cgroup_per_zone *mz_node; 417 418 if (mz->on_tree) 419 return; 420 421 mz->usage_in_excess = new_usage_in_excess; 422 if (!mz->usage_in_excess) 423 return; 424 while (*p) { 425 parent = *p; 426 mz_node = rb_entry(parent, struct mem_cgroup_per_zone, 427 tree_node); 428 if (mz->usage_in_excess < mz_node->usage_in_excess) 429 p = &(*p)->rb_left; 430 /* 431 * We can't avoid mem cgroups that are over their soft 432 * limit by the same amount 433 */ 434 else if (mz->usage_in_excess >= mz_node->usage_in_excess) 435 p = &(*p)->rb_right; 436 } 437 rb_link_node(&mz->tree_node, parent, p); 438 rb_insert_color(&mz->tree_node, &mctz->rb_root); 439 mz->on_tree = true; 440 } 441 442 static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_zone *mz, 443 struct mem_cgroup_tree_per_zone *mctz) 444 { 445 if (!mz->on_tree) 446 return; 447 rb_erase(&mz->tree_node, &mctz->rb_root); 448 mz->on_tree = false; 449 } 450 451 static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_zone *mz, 452 struct mem_cgroup_tree_per_zone *mctz) 453 { 454 unsigned long flags; 455 456 spin_lock_irqsave(&mctz->lock, flags); 457 __mem_cgroup_remove_exceeded(mz, mctz); 458 spin_unlock_irqrestore(&mctz->lock, flags); 459 } 460 461 static unsigned long soft_limit_excess(struct mem_cgroup *memcg) 462 { 463 unsigned long nr_pages = page_counter_read(&memcg->memory); 464 unsigned long soft_limit = READ_ONCE(memcg->soft_limit); 465 unsigned long excess = 0; 466 467 if (nr_pages > soft_limit) 468 excess = nr_pages - soft_limit; 469 470 return excess; 471 } 472 473 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page) 474 { 475 unsigned long excess; 476 struct mem_cgroup_per_zone *mz; 477 struct mem_cgroup_tree_per_zone *mctz; 478 479 mctz = soft_limit_tree_from_page(page); 480 /* 481 * Necessary to update all ancestors when hierarchy is used. 482 * because their event counter is not touched. 483 */ 484 for (; memcg; memcg = parent_mem_cgroup(memcg)) { 485 mz = mem_cgroup_page_zoneinfo(memcg, page); 486 excess = soft_limit_excess(memcg); 487 /* 488 * We have to update the tree if mz is on RB-tree or 489 * mem is over its softlimit. 490 */ 491 if (excess || mz->on_tree) { 492 unsigned long flags; 493 494 spin_lock_irqsave(&mctz->lock, flags); 495 /* if on-tree, remove it */ 496 if (mz->on_tree) 497 __mem_cgroup_remove_exceeded(mz, mctz); 498 /* 499 * Insert again. mz->usage_in_excess will be updated. 500 * If excess is 0, no tree ops. 501 */ 502 __mem_cgroup_insert_exceeded(mz, mctz, excess); 503 spin_unlock_irqrestore(&mctz->lock, flags); 504 } 505 } 506 } 507 508 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg) 509 { 510 struct mem_cgroup_tree_per_zone *mctz; 511 struct mem_cgroup_per_zone *mz; 512 int nid, zid; 513 514 for_each_node(nid) { 515 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 516 mz = &memcg->nodeinfo[nid]->zoneinfo[zid]; 517 mctz = soft_limit_tree_node_zone(nid, zid); 518 mem_cgroup_remove_exceeded(mz, mctz); 519 } 520 } 521 } 522 523 static struct mem_cgroup_per_zone * 524 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz) 525 { 526 struct rb_node *rightmost = NULL; 527 struct mem_cgroup_per_zone *mz; 528 529 retry: 530 mz = NULL; 531 rightmost = rb_last(&mctz->rb_root); 532 if (!rightmost) 533 goto done; /* Nothing to reclaim from */ 534 535 mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node); 536 /* 537 * Remove the node now but someone else can add it back, 538 * we will to add it back at the end of reclaim to its correct 539 * position in the tree. 540 */ 541 __mem_cgroup_remove_exceeded(mz, mctz); 542 if (!soft_limit_excess(mz->memcg) || 543 !css_tryget_online(&mz->memcg->css)) 544 goto retry; 545 done: 546 return mz; 547 } 548 549 static struct mem_cgroup_per_zone * 550 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz) 551 { 552 struct mem_cgroup_per_zone *mz; 553 554 spin_lock_irq(&mctz->lock); 555 mz = __mem_cgroup_largest_soft_limit_node(mctz); 556 spin_unlock_irq(&mctz->lock); 557 return mz; 558 } 559 560 /* 561 * Return page count for single (non recursive) @memcg. 562 * 563 * Implementation Note: reading percpu statistics for memcg. 564 * 565 * Both of vmstat[] and percpu_counter has threshold and do periodic 566 * synchronization to implement "quick" read. There are trade-off between 567 * reading cost and precision of value. Then, we may have a chance to implement 568 * a periodic synchronization of counter in memcg's counter. 569 * 570 * But this _read() function is used for user interface now. The user accounts 571 * memory usage by memory cgroup and he _always_ requires exact value because 572 * he accounts memory. Even if we provide quick-and-fuzzy read, we always 573 * have to visit all online cpus and make sum. So, for now, unnecessary 574 * synchronization is not implemented. (just implemented for cpu hotplug) 575 * 576 * If there are kernel internal actions which can make use of some not-exact 577 * value, and reading all cpu value can be performance bottleneck in some 578 * common workload, threshold and synchronization as vmstat[] should be 579 * implemented. 580 */ 581 static unsigned long 582 mem_cgroup_read_stat(struct mem_cgroup *memcg, enum mem_cgroup_stat_index idx) 583 { 584 long val = 0; 585 int cpu; 586 587 /* Per-cpu values can be negative, use a signed accumulator */ 588 for_each_possible_cpu(cpu) 589 val += per_cpu(memcg->stat->count[idx], cpu); 590 /* 591 * Summing races with updates, so val may be negative. Avoid exposing 592 * transient negative values. 593 */ 594 if (val < 0) 595 val = 0; 596 return val; 597 } 598 599 static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg, 600 enum mem_cgroup_events_index idx) 601 { 602 unsigned long val = 0; 603 int cpu; 604 605 for_each_possible_cpu(cpu) 606 val += per_cpu(memcg->stat->events[idx], cpu); 607 return val; 608 } 609 610 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg, 611 struct page *page, 612 bool compound, int nr_pages) 613 { 614 /* 615 * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is 616 * counted as CACHE even if it's on ANON LRU. 617 */ 618 if (PageAnon(page)) 619 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS], 620 nr_pages); 621 else 622 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE], 623 nr_pages); 624 625 if (compound) { 626 VM_BUG_ON_PAGE(!PageTransHuge(page), page); 627 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], 628 nr_pages); 629 } 630 631 /* pagein of a big page is an event. So, ignore page size */ 632 if (nr_pages > 0) 633 __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]); 634 else { 635 __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]); 636 nr_pages = -nr_pages; /* for event */ 637 } 638 639 __this_cpu_add(memcg->stat->nr_page_events, nr_pages); 640 } 641 642 unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg, 643 int nid, unsigned int lru_mask) 644 { 645 unsigned long nr = 0; 646 int zid; 647 648 VM_BUG_ON((unsigned)nid >= nr_node_ids); 649 650 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 651 struct mem_cgroup_per_zone *mz; 652 enum lru_list lru; 653 654 for_each_lru(lru) { 655 if (!(BIT(lru) & lru_mask)) 656 continue; 657 mz = &memcg->nodeinfo[nid]->zoneinfo[zid]; 658 nr += mz->lru_size[lru]; 659 } 660 } 661 return nr; 662 } 663 664 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg, 665 unsigned int lru_mask) 666 { 667 unsigned long nr = 0; 668 int nid; 669 670 for_each_node_state(nid, N_MEMORY) 671 nr += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask); 672 return nr; 673 } 674 675 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg, 676 enum mem_cgroup_events_target target) 677 { 678 unsigned long val, next; 679 680 val = __this_cpu_read(memcg->stat->nr_page_events); 681 next = __this_cpu_read(memcg->stat->targets[target]); 682 /* from time_after() in jiffies.h */ 683 if ((long)next - (long)val < 0) { 684 switch (target) { 685 case MEM_CGROUP_TARGET_THRESH: 686 next = val + THRESHOLDS_EVENTS_TARGET; 687 break; 688 case MEM_CGROUP_TARGET_SOFTLIMIT: 689 next = val + SOFTLIMIT_EVENTS_TARGET; 690 break; 691 case MEM_CGROUP_TARGET_NUMAINFO: 692 next = val + NUMAINFO_EVENTS_TARGET; 693 break; 694 default: 695 break; 696 } 697 __this_cpu_write(memcg->stat->targets[target], next); 698 return true; 699 } 700 return false; 701 } 702 703 /* 704 * Check events in order. 705 * 706 */ 707 static void memcg_check_events(struct mem_cgroup *memcg, struct page *page) 708 { 709 /* threshold event is triggered in finer grain than soft limit */ 710 if (unlikely(mem_cgroup_event_ratelimit(memcg, 711 MEM_CGROUP_TARGET_THRESH))) { 712 bool do_softlimit; 713 bool do_numainfo __maybe_unused; 714 715 do_softlimit = mem_cgroup_event_ratelimit(memcg, 716 MEM_CGROUP_TARGET_SOFTLIMIT); 717 #if MAX_NUMNODES > 1 718 do_numainfo = mem_cgroup_event_ratelimit(memcg, 719 MEM_CGROUP_TARGET_NUMAINFO); 720 #endif 721 mem_cgroup_threshold(memcg); 722 if (unlikely(do_softlimit)) 723 mem_cgroup_update_tree(memcg, page); 724 #if MAX_NUMNODES > 1 725 if (unlikely(do_numainfo)) 726 atomic_inc(&memcg->numainfo_events); 727 #endif 728 } 729 } 730 731 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) 732 { 733 /* 734 * mm_update_next_owner() may clear mm->owner to NULL 735 * if it races with swapoff, page migration, etc. 736 * So this can be called with p == NULL. 737 */ 738 if (unlikely(!p)) 739 return NULL; 740 741 return mem_cgroup_from_css(task_css(p, memory_cgrp_id)); 742 } 743 EXPORT_SYMBOL(mem_cgroup_from_task); 744 745 static struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm) 746 { 747 struct mem_cgroup *memcg = NULL; 748 749 rcu_read_lock(); 750 do { 751 /* 752 * Page cache insertions can happen withou an 753 * actual mm context, e.g. during disk probing 754 * on boot, loopback IO, acct() writes etc. 755 */ 756 if (unlikely(!mm)) 757 memcg = root_mem_cgroup; 758 else { 759 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); 760 if (unlikely(!memcg)) 761 memcg = root_mem_cgroup; 762 } 763 } while (!css_tryget_online(&memcg->css)); 764 rcu_read_unlock(); 765 return memcg; 766 } 767 768 /** 769 * mem_cgroup_iter - iterate over memory cgroup hierarchy 770 * @root: hierarchy root 771 * @prev: previously returned memcg, NULL on first invocation 772 * @reclaim: cookie for shared reclaim walks, NULL for full walks 773 * 774 * Returns references to children of the hierarchy below @root, or 775 * @root itself, or %NULL after a full round-trip. 776 * 777 * Caller must pass the return value in @prev on subsequent 778 * invocations for reference counting, or use mem_cgroup_iter_break() 779 * to cancel a hierarchy walk before the round-trip is complete. 780 * 781 * Reclaimers can specify a zone and a priority level in @reclaim to 782 * divide up the memcgs in the hierarchy among all concurrent 783 * reclaimers operating on the same zone and priority. 784 */ 785 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root, 786 struct mem_cgroup *prev, 787 struct mem_cgroup_reclaim_cookie *reclaim) 788 { 789 struct mem_cgroup_reclaim_iter *uninitialized_var(iter); 790 struct cgroup_subsys_state *css = NULL; 791 struct mem_cgroup *memcg = NULL; 792 struct mem_cgroup *pos = NULL; 793 794 if (mem_cgroup_disabled()) 795 return NULL; 796 797 if (!root) 798 root = root_mem_cgroup; 799 800 if (prev && !reclaim) 801 pos = prev; 802 803 if (!root->use_hierarchy && root != root_mem_cgroup) { 804 if (prev) 805 goto out; 806 return root; 807 } 808 809 rcu_read_lock(); 810 811 if (reclaim) { 812 struct mem_cgroup_per_zone *mz; 813 814 mz = mem_cgroup_zone_zoneinfo(root, reclaim->zone); 815 iter = &mz->iter[reclaim->priority]; 816 817 if (prev && reclaim->generation != iter->generation) 818 goto out_unlock; 819 820 while (1) { 821 pos = READ_ONCE(iter->position); 822 if (!pos || css_tryget(&pos->css)) 823 break; 824 /* 825 * css reference reached zero, so iter->position will 826 * be cleared by ->css_released. However, we should not 827 * rely on this happening soon, because ->css_released 828 * is called from a work queue, and by busy-waiting we 829 * might block it. So we clear iter->position right 830 * away. 831 */ 832 (void)cmpxchg(&iter->position, pos, NULL); 833 } 834 } 835 836 if (pos) 837 css = &pos->css; 838 839 for (;;) { 840 css = css_next_descendant_pre(css, &root->css); 841 if (!css) { 842 /* 843 * Reclaimers share the hierarchy walk, and a 844 * new one might jump in right at the end of 845 * the hierarchy - make sure they see at least 846 * one group and restart from the beginning. 847 */ 848 if (!prev) 849 continue; 850 break; 851 } 852 853 /* 854 * Verify the css and acquire a reference. The root 855 * is provided by the caller, so we know it's alive 856 * and kicking, and don't take an extra reference. 857 */ 858 memcg = mem_cgroup_from_css(css); 859 860 if (css == &root->css) 861 break; 862 863 if (css_tryget(css)) 864 break; 865 866 memcg = NULL; 867 } 868 869 if (reclaim) { 870 /* 871 * The position could have already been updated by a competing 872 * thread, so check that the value hasn't changed since we read 873 * it to avoid reclaiming from the same cgroup twice. 874 */ 875 (void)cmpxchg(&iter->position, pos, memcg); 876 877 if (pos) 878 css_put(&pos->css); 879 880 if (!memcg) 881 iter->generation++; 882 else if (!prev) 883 reclaim->generation = iter->generation; 884 } 885 886 out_unlock: 887 rcu_read_unlock(); 888 out: 889 if (prev && prev != root) 890 css_put(&prev->css); 891 892 return memcg; 893 } 894 895 /** 896 * mem_cgroup_iter_break - abort a hierarchy walk prematurely 897 * @root: hierarchy root 898 * @prev: last visited hierarchy member as returned by mem_cgroup_iter() 899 */ 900 void mem_cgroup_iter_break(struct mem_cgroup *root, 901 struct mem_cgroup *prev) 902 { 903 if (!root) 904 root = root_mem_cgroup; 905 if (prev && prev != root) 906 css_put(&prev->css); 907 } 908 909 static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg) 910 { 911 struct mem_cgroup *memcg = dead_memcg; 912 struct mem_cgroup_reclaim_iter *iter; 913 struct mem_cgroup_per_zone *mz; 914 int nid, zid; 915 int i; 916 917 while ((memcg = parent_mem_cgroup(memcg))) { 918 for_each_node(nid) { 919 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 920 mz = &memcg->nodeinfo[nid]->zoneinfo[zid]; 921 for (i = 0; i <= DEF_PRIORITY; i++) { 922 iter = &mz->iter[i]; 923 cmpxchg(&iter->position, 924 dead_memcg, NULL); 925 } 926 } 927 } 928 } 929 } 930 931 /* 932 * Iteration constructs for visiting all cgroups (under a tree). If 933 * loops are exited prematurely (break), mem_cgroup_iter_break() must 934 * be used for reference counting. 935 */ 936 #define for_each_mem_cgroup_tree(iter, root) \ 937 for (iter = mem_cgroup_iter(root, NULL, NULL); \ 938 iter != NULL; \ 939 iter = mem_cgroup_iter(root, iter, NULL)) 940 941 #define for_each_mem_cgroup(iter) \ 942 for (iter = mem_cgroup_iter(NULL, NULL, NULL); \ 943 iter != NULL; \ 944 iter = mem_cgroup_iter(NULL, iter, NULL)) 945 946 /** 947 * mem_cgroup_zone_lruvec - get the lru list vector for a zone and memcg 948 * @zone: zone of the wanted lruvec 949 * @memcg: memcg of the wanted lruvec 950 * 951 * Returns the lru list vector holding pages for the given @zone and 952 * @mem. This can be the global zone lruvec, if the memory controller 953 * is disabled. 954 */ 955 struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone, 956 struct mem_cgroup *memcg) 957 { 958 struct mem_cgroup_per_zone *mz; 959 struct lruvec *lruvec; 960 961 if (mem_cgroup_disabled()) { 962 lruvec = &zone->lruvec; 963 goto out; 964 } 965 966 mz = mem_cgroup_zone_zoneinfo(memcg, zone); 967 lruvec = &mz->lruvec; 968 out: 969 /* 970 * Since a node can be onlined after the mem_cgroup was created, 971 * we have to be prepared to initialize lruvec->zone here; 972 * and if offlined then reonlined, we need to reinitialize it. 973 */ 974 if (unlikely(lruvec->zone != zone)) 975 lruvec->zone = zone; 976 return lruvec; 977 } 978 979 /** 980 * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page 981 * @page: the page 982 * @zone: zone of the page 983 * 984 * This function is only safe when following the LRU page isolation 985 * and putback protocol: the LRU lock must be held, and the page must 986 * either be PageLRU() or the caller must have isolated/allocated it. 987 */ 988 struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct zone *zone) 989 { 990 struct mem_cgroup_per_zone *mz; 991 struct mem_cgroup *memcg; 992 struct lruvec *lruvec; 993 994 if (mem_cgroup_disabled()) { 995 lruvec = &zone->lruvec; 996 goto out; 997 } 998 999 memcg = page->mem_cgroup; 1000 /* 1001 * Swapcache readahead pages are added to the LRU - and 1002 * possibly migrated - before they are charged. 1003 */ 1004 if (!memcg) 1005 memcg = root_mem_cgroup; 1006 1007 mz = mem_cgroup_page_zoneinfo(memcg, page); 1008 lruvec = &mz->lruvec; 1009 out: 1010 /* 1011 * Since a node can be onlined after the mem_cgroup was created, 1012 * we have to be prepared to initialize lruvec->zone here; 1013 * and if offlined then reonlined, we need to reinitialize it. 1014 */ 1015 if (unlikely(lruvec->zone != zone)) 1016 lruvec->zone = zone; 1017 return lruvec; 1018 } 1019 1020 /** 1021 * mem_cgroup_update_lru_size - account for adding or removing an lru page 1022 * @lruvec: mem_cgroup per zone lru vector 1023 * @lru: index of lru list the page is sitting on 1024 * @nr_pages: positive when adding or negative when removing 1025 * 1026 * This function must be called when a page is added to or removed from an 1027 * lru list. 1028 */ 1029 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, 1030 int nr_pages) 1031 { 1032 struct mem_cgroup_per_zone *mz; 1033 unsigned long *lru_size; 1034 1035 if (mem_cgroup_disabled()) 1036 return; 1037 1038 mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec); 1039 lru_size = mz->lru_size + lru; 1040 *lru_size += nr_pages; 1041 VM_BUG_ON((long)(*lru_size) < 0); 1042 } 1043 1044 bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg) 1045 { 1046 struct mem_cgroup *task_memcg; 1047 struct task_struct *p; 1048 bool ret; 1049 1050 p = find_lock_task_mm(task); 1051 if (p) { 1052 task_memcg = get_mem_cgroup_from_mm(p->mm); 1053 task_unlock(p); 1054 } else { 1055 /* 1056 * All threads may have already detached their mm's, but the oom 1057 * killer still needs to detect if they have already been oom 1058 * killed to prevent needlessly killing additional tasks. 1059 */ 1060 rcu_read_lock(); 1061 task_memcg = mem_cgroup_from_task(task); 1062 css_get(&task_memcg->css); 1063 rcu_read_unlock(); 1064 } 1065 ret = mem_cgroup_is_descendant(task_memcg, memcg); 1066 css_put(&task_memcg->css); 1067 return ret; 1068 } 1069 1070 /** 1071 * mem_cgroup_margin - calculate chargeable space of a memory cgroup 1072 * @memcg: the memory cgroup 1073 * 1074 * Returns the maximum amount of memory @mem can be charged with, in 1075 * pages. 1076 */ 1077 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg) 1078 { 1079 unsigned long margin = 0; 1080 unsigned long count; 1081 unsigned long limit; 1082 1083 count = page_counter_read(&memcg->memory); 1084 limit = READ_ONCE(memcg->memory.limit); 1085 if (count < limit) 1086 margin = limit - count; 1087 1088 if (do_memsw_account()) { 1089 count = page_counter_read(&memcg->memsw); 1090 limit = READ_ONCE(memcg->memsw.limit); 1091 if (count <= limit) 1092 margin = min(margin, limit - count); 1093 } 1094 1095 return margin; 1096 } 1097 1098 /* 1099 * A routine for checking "mem" is under move_account() or not. 1100 * 1101 * Checking a cgroup is mc.from or mc.to or under hierarchy of 1102 * moving cgroups. This is for waiting at high-memory pressure 1103 * caused by "move". 1104 */ 1105 static bool mem_cgroup_under_move(struct mem_cgroup *memcg) 1106 { 1107 struct mem_cgroup *from; 1108 struct mem_cgroup *to; 1109 bool ret = false; 1110 /* 1111 * Unlike task_move routines, we access mc.to, mc.from not under 1112 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead. 1113 */ 1114 spin_lock(&mc.lock); 1115 from = mc.from; 1116 to = mc.to; 1117 if (!from) 1118 goto unlock; 1119 1120 ret = mem_cgroup_is_descendant(from, memcg) || 1121 mem_cgroup_is_descendant(to, memcg); 1122 unlock: 1123 spin_unlock(&mc.lock); 1124 return ret; 1125 } 1126 1127 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg) 1128 { 1129 if (mc.moving_task && current != mc.moving_task) { 1130 if (mem_cgroup_under_move(memcg)) { 1131 DEFINE_WAIT(wait); 1132 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE); 1133 /* moving charge context might have finished. */ 1134 if (mc.moving_task) 1135 schedule(); 1136 finish_wait(&mc.waitq, &wait); 1137 return true; 1138 } 1139 } 1140 return false; 1141 } 1142 1143 #define K(x) ((x) << (PAGE_SHIFT-10)) 1144 /** 1145 * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller. 1146 * @memcg: The memory cgroup that went over limit 1147 * @p: Task that is going to be killed 1148 * 1149 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is 1150 * enabled 1151 */ 1152 void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p) 1153 { 1154 struct mem_cgroup *iter; 1155 unsigned int i; 1156 1157 rcu_read_lock(); 1158 1159 if (p) { 1160 pr_info("Task in "); 1161 pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id)); 1162 pr_cont(" killed as a result of limit of "); 1163 } else { 1164 pr_info("Memory limit reached of cgroup "); 1165 } 1166 1167 pr_cont_cgroup_path(memcg->css.cgroup); 1168 pr_cont("\n"); 1169 1170 rcu_read_unlock(); 1171 1172 pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n", 1173 K((u64)page_counter_read(&memcg->memory)), 1174 K((u64)memcg->memory.limit), memcg->memory.failcnt); 1175 pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n", 1176 K((u64)page_counter_read(&memcg->memsw)), 1177 K((u64)memcg->memsw.limit), memcg->memsw.failcnt); 1178 pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n", 1179 K((u64)page_counter_read(&memcg->kmem)), 1180 K((u64)memcg->kmem.limit), memcg->kmem.failcnt); 1181 1182 for_each_mem_cgroup_tree(iter, memcg) { 1183 pr_info("Memory cgroup stats for "); 1184 pr_cont_cgroup_path(iter->css.cgroup); 1185 pr_cont(":"); 1186 1187 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) { 1188 if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account) 1189 continue; 1190 pr_cont(" %s:%luKB", mem_cgroup_stat_names[i], 1191 K(mem_cgroup_read_stat(iter, i))); 1192 } 1193 1194 for (i = 0; i < NR_LRU_LISTS; i++) 1195 pr_cont(" %s:%luKB", mem_cgroup_lru_names[i], 1196 K(mem_cgroup_nr_lru_pages(iter, BIT(i)))); 1197 1198 pr_cont("\n"); 1199 } 1200 } 1201 1202 /* 1203 * This function returns the number of memcg under hierarchy tree. Returns 1204 * 1(self count) if no children. 1205 */ 1206 static int mem_cgroup_count_children(struct mem_cgroup *memcg) 1207 { 1208 int num = 0; 1209 struct mem_cgroup *iter; 1210 1211 for_each_mem_cgroup_tree(iter, memcg) 1212 num++; 1213 return num; 1214 } 1215 1216 /* 1217 * Return the memory (and swap, if configured) limit for a memcg. 1218 */ 1219 static unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg) 1220 { 1221 unsigned long limit; 1222 1223 limit = memcg->memory.limit; 1224 if (mem_cgroup_swappiness(memcg)) { 1225 unsigned long memsw_limit; 1226 unsigned long swap_limit; 1227 1228 memsw_limit = memcg->memsw.limit; 1229 swap_limit = memcg->swap.limit; 1230 swap_limit = min(swap_limit, (unsigned long)total_swap_pages); 1231 limit = min(limit + swap_limit, memsw_limit); 1232 } 1233 return limit; 1234 } 1235 1236 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask, 1237 int order) 1238 { 1239 struct oom_control oc = { 1240 .zonelist = NULL, 1241 .nodemask = NULL, 1242 .gfp_mask = gfp_mask, 1243 .order = order, 1244 }; 1245 struct mem_cgroup *iter; 1246 unsigned long chosen_points = 0; 1247 unsigned long totalpages; 1248 unsigned int points = 0; 1249 struct task_struct *chosen = NULL; 1250 1251 mutex_lock(&oom_lock); 1252 1253 /* 1254 * If current has a pending SIGKILL or is exiting, then automatically 1255 * select it. The goal is to allow it to allocate so that it may 1256 * quickly exit and free its memory. 1257 */ 1258 if (fatal_signal_pending(current) || task_will_free_mem(current)) { 1259 mark_oom_victim(current); 1260 goto unlock; 1261 } 1262 1263 check_panic_on_oom(&oc, CONSTRAINT_MEMCG, memcg); 1264 totalpages = mem_cgroup_get_limit(memcg) ? : 1; 1265 for_each_mem_cgroup_tree(iter, memcg) { 1266 struct css_task_iter it; 1267 struct task_struct *task; 1268 1269 css_task_iter_start(&iter->css, &it); 1270 while ((task = css_task_iter_next(&it))) { 1271 switch (oom_scan_process_thread(&oc, task, totalpages)) { 1272 case OOM_SCAN_SELECT: 1273 if (chosen) 1274 put_task_struct(chosen); 1275 chosen = task; 1276 chosen_points = ULONG_MAX; 1277 get_task_struct(chosen); 1278 /* fall through */ 1279 case OOM_SCAN_CONTINUE: 1280 continue; 1281 case OOM_SCAN_ABORT: 1282 css_task_iter_end(&it); 1283 mem_cgroup_iter_break(memcg, iter); 1284 if (chosen) 1285 put_task_struct(chosen); 1286 goto unlock; 1287 case OOM_SCAN_OK: 1288 break; 1289 }; 1290 points = oom_badness(task, memcg, NULL, totalpages); 1291 if (!points || points < chosen_points) 1292 continue; 1293 /* Prefer thread group leaders for display purposes */ 1294 if (points == chosen_points && 1295 thread_group_leader(chosen)) 1296 continue; 1297 1298 if (chosen) 1299 put_task_struct(chosen); 1300 chosen = task; 1301 chosen_points = points; 1302 get_task_struct(chosen); 1303 } 1304 css_task_iter_end(&it); 1305 } 1306 1307 if (chosen) { 1308 points = chosen_points * 1000 / totalpages; 1309 oom_kill_process(&oc, chosen, points, totalpages, memcg, 1310 "Memory cgroup out of memory"); 1311 } 1312 unlock: 1313 mutex_unlock(&oom_lock); 1314 return chosen; 1315 } 1316 1317 #if MAX_NUMNODES > 1 1318 1319 /** 1320 * test_mem_cgroup_node_reclaimable 1321 * @memcg: the target memcg 1322 * @nid: the node ID to be checked. 1323 * @noswap : specify true here if the user wants flle only information. 1324 * 1325 * This function returns whether the specified memcg contains any 1326 * reclaimable pages on a node. Returns true if there are any reclaimable 1327 * pages in the node. 1328 */ 1329 static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg, 1330 int nid, bool noswap) 1331 { 1332 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE)) 1333 return true; 1334 if (noswap || !total_swap_pages) 1335 return false; 1336 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON)) 1337 return true; 1338 return false; 1339 1340 } 1341 1342 /* 1343 * Always updating the nodemask is not very good - even if we have an empty 1344 * list or the wrong list here, we can start from some node and traverse all 1345 * nodes based on the zonelist. So update the list loosely once per 10 secs. 1346 * 1347 */ 1348 static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg) 1349 { 1350 int nid; 1351 /* 1352 * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET 1353 * pagein/pageout changes since the last update. 1354 */ 1355 if (!atomic_read(&memcg->numainfo_events)) 1356 return; 1357 if (atomic_inc_return(&memcg->numainfo_updating) > 1) 1358 return; 1359 1360 /* make a nodemask where this memcg uses memory from */ 1361 memcg->scan_nodes = node_states[N_MEMORY]; 1362 1363 for_each_node_mask(nid, node_states[N_MEMORY]) { 1364 1365 if (!test_mem_cgroup_node_reclaimable(memcg, nid, false)) 1366 node_clear(nid, memcg->scan_nodes); 1367 } 1368 1369 atomic_set(&memcg->numainfo_events, 0); 1370 atomic_set(&memcg->numainfo_updating, 0); 1371 } 1372 1373 /* 1374 * Selecting a node where we start reclaim from. Because what we need is just 1375 * reducing usage counter, start from anywhere is O,K. Considering 1376 * memory reclaim from current node, there are pros. and cons. 1377 * 1378 * Freeing memory from current node means freeing memory from a node which 1379 * we'll use or we've used. So, it may make LRU bad. And if several threads 1380 * hit limits, it will see a contention on a node. But freeing from remote 1381 * node means more costs for memory reclaim because of memory latency. 1382 * 1383 * Now, we use round-robin. Better algorithm is welcomed. 1384 */ 1385 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg) 1386 { 1387 int node; 1388 1389 mem_cgroup_may_update_nodemask(memcg); 1390 node = memcg->last_scanned_node; 1391 1392 node = next_node(node, memcg->scan_nodes); 1393 if (node == MAX_NUMNODES) 1394 node = first_node(memcg->scan_nodes); 1395 /* 1396 * We call this when we hit limit, not when pages are added to LRU. 1397 * No LRU may hold pages because all pages are UNEVICTABLE or 1398 * memcg is too small and all pages are not on LRU. In that case, 1399 * we use curret node. 1400 */ 1401 if (unlikely(node == MAX_NUMNODES)) 1402 node = numa_node_id(); 1403 1404 memcg->last_scanned_node = node; 1405 return node; 1406 } 1407 #else 1408 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg) 1409 { 1410 return 0; 1411 } 1412 #endif 1413 1414 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg, 1415 struct zone *zone, 1416 gfp_t gfp_mask, 1417 unsigned long *total_scanned) 1418 { 1419 struct mem_cgroup *victim = NULL; 1420 int total = 0; 1421 int loop = 0; 1422 unsigned long excess; 1423 unsigned long nr_scanned; 1424 struct mem_cgroup_reclaim_cookie reclaim = { 1425 .zone = zone, 1426 .priority = 0, 1427 }; 1428 1429 excess = soft_limit_excess(root_memcg); 1430 1431 while (1) { 1432 victim = mem_cgroup_iter(root_memcg, victim, &reclaim); 1433 if (!victim) { 1434 loop++; 1435 if (loop >= 2) { 1436 /* 1437 * If we have not been able to reclaim 1438 * anything, it might because there are 1439 * no reclaimable pages under this hierarchy 1440 */ 1441 if (!total) 1442 break; 1443 /* 1444 * We want to do more targeted reclaim. 1445 * excess >> 2 is not to excessive so as to 1446 * reclaim too much, nor too less that we keep 1447 * coming back to reclaim from this cgroup 1448 */ 1449 if (total >= (excess >> 2) || 1450 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) 1451 break; 1452 } 1453 continue; 1454 } 1455 total += mem_cgroup_shrink_node_zone(victim, gfp_mask, false, 1456 zone, &nr_scanned); 1457 *total_scanned += nr_scanned; 1458 if (!soft_limit_excess(root_memcg)) 1459 break; 1460 } 1461 mem_cgroup_iter_break(root_memcg, victim); 1462 return total; 1463 } 1464 1465 #ifdef CONFIG_LOCKDEP 1466 static struct lockdep_map memcg_oom_lock_dep_map = { 1467 .name = "memcg_oom_lock", 1468 }; 1469 #endif 1470 1471 static DEFINE_SPINLOCK(memcg_oom_lock); 1472 1473 /* 1474 * Check OOM-Killer is already running under our hierarchy. 1475 * If someone is running, return false. 1476 */ 1477 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg) 1478 { 1479 struct mem_cgroup *iter, *failed = NULL; 1480 1481 spin_lock(&memcg_oom_lock); 1482 1483 for_each_mem_cgroup_tree(iter, memcg) { 1484 if (iter->oom_lock) { 1485 /* 1486 * this subtree of our hierarchy is already locked 1487 * so we cannot give a lock. 1488 */ 1489 failed = iter; 1490 mem_cgroup_iter_break(memcg, iter); 1491 break; 1492 } else 1493 iter->oom_lock = true; 1494 } 1495 1496 if (failed) { 1497 /* 1498 * OK, we failed to lock the whole subtree so we have 1499 * to clean up what we set up to the failing subtree 1500 */ 1501 for_each_mem_cgroup_tree(iter, memcg) { 1502 if (iter == failed) { 1503 mem_cgroup_iter_break(memcg, iter); 1504 break; 1505 } 1506 iter->oom_lock = false; 1507 } 1508 } else 1509 mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_); 1510 1511 spin_unlock(&memcg_oom_lock); 1512 1513 return !failed; 1514 } 1515 1516 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg) 1517 { 1518 struct mem_cgroup *iter; 1519 1520 spin_lock(&memcg_oom_lock); 1521 mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_); 1522 for_each_mem_cgroup_tree(iter, memcg) 1523 iter->oom_lock = false; 1524 spin_unlock(&memcg_oom_lock); 1525 } 1526 1527 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg) 1528 { 1529 struct mem_cgroup *iter; 1530 1531 spin_lock(&memcg_oom_lock); 1532 for_each_mem_cgroup_tree(iter, memcg) 1533 iter->under_oom++; 1534 spin_unlock(&memcg_oom_lock); 1535 } 1536 1537 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg) 1538 { 1539 struct mem_cgroup *iter; 1540 1541 /* 1542 * When a new child is created while the hierarchy is under oom, 1543 * mem_cgroup_oom_lock() may not be called. Watch for underflow. 1544 */ 1545 spin_lock(&memcg_oom_lock); 1546 for_each_mem_cgroup_tree(iter, memcg) 1547 if (iter->under_oom > 0) 1548 iter->under_oom--; 1549 spin_unlock(&memcg_oom_lock); 1550 } 1551 1552 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq); 1553 1554 struct oom_wait_info { 1555 struct mem_cgroup *memcg; 1556 wait_queue_t wait; 1557 }; 1558 1559 static int memcg_oom_wake_function(wait_queue_t *wait, 1560 unsigned mode, int sync, void *arg) 1561 { 1562 struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg; 1563 struct mem_cgroup *oom_wait_memcg; 1564 struct oom_wait_info *oom_wait_info; 1565 1566 oom_wait_info = container_of(wait, struct oom_wait_info, wait); 1567 oom_wait_memcg = oom_wait_info->memcg; 1568 1569 if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) && 1570 !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg)) 1571 return 0; 1572 return autoremove_wake_function(wait, mode, sync, arg); 1573 } 1574 1575 static void memcg_oom_recover(struct mem_cgroup *memcg) 1576 { 1577 /* 1578 * For the following lockless ->under_oom test, the only required 1579 * guarantee is that it must see the state asserted by an OOM when 1580 * this function is called as a result of userland actions 1581 * triggered by the notification of the OOM. This is trivially 1582 * achieved by invoking mem_cgroup_mark_under_oom() before 1583 * triggering notification. 1584 */ 1585 if (memcg && memcg->under_oom) 1586 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg); 1587 } 1588 1589 static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order) 1590 { 1591 if (!current->memcg_may_oom) 1592 return; 1593 /* 1594 * We are in the middle of the charge context here, so we 1595 * don't want to block when potentially sitting on a callstack 1596 * that holds all kinds of filesystem and mm locks. 1597 * 1598 * Also, the caller may handle a failed allocation gracefully 1599 * (like optional page cache readahead) and so an OOM killer 1600 * invocation might not even be necessary. 1601 * 1602 * That's why we don't do anything here except remember the 1603 * OOM context and then deal with it at the end of the page 1604 * fault when the stack is unwound, the locks are released, 1605 * and when we know whether the fault was overall successful. 1606 */ 1607 css_get(&memcg->css); 1608 current->memcg_in_oom = memcg; 1609 current->memcg_oom_gfp_mask = mask; 1610 current->memcg_oom_order = order; 1611 } 1612 1613 /** 1614 * mem_cgroup_oom_synchronize - complete memcg OOM handling 1615 * @handle: actually kill/wait or just clean up the OOM state 1616 * 1617 * This has to be called at the end of a page fault if the memcg OOM 1618 * handler was enabled. 1619 * 1620 * Memcg supports userspace OOM handling where failed allocations must 1621 * sleep on a waitqueue until the userspace task resolves the 1622 * situation. Sleeping directly in the charge context with all kinds 1623 * of locks held is not a good idea, instead we remember an OOM state 1624 * in the task and mem_cgroup_oom_synchronize() has to be called at 1625 * the end of the page fault to complete the OOM handling. 1626 * 1627 * Returns %true if an ongoing memcg OOM situation was detected and 1628 * completed, %false otherwise. 1629 */ 1630 bool mem_cgroup_oom_synchronize(bool handle) 1631 { 1632 struct mem_cgroup *memcg = current->memcg_in_oom; 1633 struct oom_wait_info owait; 1634 bool locked; 1635 1636 /* OOM is global, do not handle */ 1637 if (!memcg) 1638 return false; 1639 1640 if (!handle || oom_killer_disabled) 1641 goto cleanup; 1642 1643 owait.memcg = memcg; 1644 owait.wait.flags = 0; 1645 owait.wait.func = memcg_oom_wake_function; 1646 owait.wait.private = current; 1647 INIT_LIST_HEAD(&owait.wait.task_list); 1648 1649 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE); 1650 mem_cgroup_mark_under_oom(memcg); 1651 1652 locked = mem_cgroup_oom_trylock(memcg); 1653 1654 if (locked) 1655 mem_cgroup_oom_notify(memcg); 1656 1657 if (locked && !memcg->oom_kill_disable) { 1658 mem_cgroup_unmark_under_oom(memcg); 1659 finish_wait(&memcg_oom_waitq, &owait.wait); 1660 mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask, 1661 current->memcg_oom_order); 1662 } else { 1663 schedule(); 1664 mem_cgroup_unmark_under_oom(memcg); 1665 finish_wait(&memcg_oom_waitq, &owait.wait); 1666 } 1667 1668 if (locked) { 1669 mem_cgroup_oom_unlock(memcg); 1670 /* 1671 * There is no guarantee that an OOM-lock contender 1672 * sees the wakeups triggered by the OOM kill 1673 * uncharges. Wake any sleepers explicitely. 1674 */ 1675 memcg_oom_recover(memcg); 1676 } 1677 cleanup: 1678 current->memcg_in_oom = NULL; 1679 css_put(&memcg->css); 1680 return true; 1681 } 1682 1683 /** 1684 * lock_page_memcg - lock a page->mem_cgroup binding 1685 * @page: the page 1686 * 1687 * This function protects unlocked LRU pages from being moved to 1688 * another cgroup and stabilizes their page->mem_cgroup binding. 1689 */ 1690 void lock_page_memcg(struct page *page) 1691 { 1692 struct mem_cgroup *memcg; 1693 unsigned long flags; 1694 1695 /* 1696 * The RCU lock is held throughout the transaction. The fast 1697 * path can get away without acquiring the memcg->move_lock 1698 * because page moving starts with an RCU grace period. 1699 */ 1700 rcu_read_lock(); 1701 1702 if (mem_cgroup_disabled()) 1703 return; 1704 again: 1705 memcg = page->mem_cgroup; 1706 if (unlikely(!memcg)) 1707 return; 1708 1709 if (atomic_read(&memcg->moving_account) <= 0) 1710 return; 1711 1712 spin_lock_irqsave(&memcg->move_lock, flags); 1713 if (memcg != page->mem_cgroup) { 1714 spin_unlock_irqrestore(&memcg->move_lock, flags); 1715 goto again; 1716 } 1717 1718 /* 1719 * When charge migration first begins, we can have locked and 1720 * unlocked page stat updates happening concurrently. Track 1721 * the task who has the lock for unlock_page_memcg(). 1722 */ 1723 memcg->move_lock_task = current; 1724 memcg->move_lock_flags = flags; 1725 1726 return; 1727 } 1728 EXPORT_SYMBOL(lock_page_memcg); 1729 1730 /** 1731 * unlock_page_memcg - unlock a page->mem_cgroup binding 1732 * @page: the page 1733 */ 1734 void unlock_page_memcg(struct page *page) 1735 { 1736 struct mem_cgroup *memcg = page->mem_cgroup; 1737 1738 if (memcg && memcg->move_lock_task == current) { 1739 unsigned long flags = memcg->move_lock_flags; 1740 1741 memcg->move_lock_task = NULL; 1742 memcg->move_lock_flags = 0; 1743 1744 spin_unlock_irqrestore(&memcg->move_lock, flags); 1745 } 1746 1747 rcu_read_unlock(); 1748 } 1749 EXPORT_SYMBOL(unlock_page_memcg); 1750 1751 /* 1752 * size of first charge trial. "32" comes from vmscan.c's magic value. 1753 * TODO: maybe necessary to use big numbers in big irons. 1754 */ 1755 #define CHARGE_BATCH 32U 1756 struct memcg_stock_pcp { 1757 struct mem_cgroup *cached; /* this never be root cgroup */ 1758 unsigned int nr_pages; 1759 struct work_struct work; 1760 unsigned long flags; 1761 #define FLUSHING_CACHED_CHARGE 0 1762 }; 1763 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock); 1764 static DEFINE_MUTEX(percpu_charge_mutex); 1765 1766 /** 1767 * consume_stock: Try to consume stocked charge on this cpu. 1768 * @memcg: memcg to consume from. 1769 * @nr_pages: how many pages to charge. 1770 * 1771 * The charges will only happen if @memcg matches the current cpu's memcg 1772 * stock, and at least @nr_pages are available in that stock. Failure to 1773 * service an allocation will refill the stock. 1774 * 1775 * returns true if successful, false otherwise. 1776 */ 1777 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) 1778 { 1779 struct memcg_stock_pcp *stock; 1780 bool ret = false; 1781 1782 if (nr_pages > CHARGE_BATCH) 1783 return ret; 1784 1785 stock = &get_cpu_var(memcg_stock); 1786 if (memcg == stock->cached && stock->nr_pages >= nr_pages) { 1787 stock->nr_pages -= nr_pages; 1788 ret = true; 1789 } 1790 put_cpu_var(memcg_stock); 1791 return ret; 1792 } 1793 1794 /* 1795 * Returns stocks cached in percpu and reset cached information. 1796 */ 1797 static void drain_stock(struct memcg_stock_pcp *stock) 1798 { 1799 struct mem_cgroup *old = stock->cached; 1800 1801 if (stock->nr_pages) { 1802 page_counter_uncharge(&old->memory, stock->nr_pages); 1803 if (do_memsw_account()) 1804 page_counter_uncharge(&old->memsw, stock->nr_pages); 1805 css_put_many(&old->css, stock->nr_pages); 1806 stock->nr_pages = 0; 1807 } 1808 stock->cached = NULL; 1809 } 1810 1811 /* 1812 * This must be called under preempt disabled or must be called by 1813 * a thread which is pinned to local cpu. 1814 */ 1815 static void drain_local_stock(struct work_struct *dummy) 1816 { 1817 struct memcg_stock_pcp *stock = this_cpu_ptr(&memcg_stock); 1818 drain_stock(stock); 1819 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags); 1820 } 1821 1822 /* 1823 * Cache charges(val) to local per_cpu area. 1824 * This will be consumed by consume_stock() function, later. 1825 */ 1826 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) 1827 { 1828 struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock); 1829 1830 if (stock->cached != memcg) { /* reset if necessary */ 1831 drain_stock(stock); 1832 stock->cached = memcg; 1833 } 1834 stock->nr_pages += nr_pages; 1835 put_cpu_var(memcg_stock); 1836 } 1837 1838 /* 1839 * Drains all per-CPU charge caches for given root_memcg resp. subtree 1840 * of the hierarchy under it. 1841 */ 1842 static void drain_all_stock(struct mem_cgroup *root_memcg) 1843 { 1844 int cpu, curcpu; 1845 1846 /* If someone's already draining, avoid adding running more workers. */ 1847 if (!mutex_trylock(&percpu_charge_mutex)) 1848 return; 1849 /* Notify other cpus that system-wide "drain" is running */ 1850 get_online_cpus(); 1851 curcpu = get_cpu(); 1852 for_each_online_cpu(cpu) { 1853 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); 1854 struct mem_cgroup *memcg; 1855 1856 memcg = stock->cached; 1857 if (!memcg || !stock->nr_pages) 1858 continue; 1859 if (!mem_cgroup_is_descendant(memcg, root_memcg)) 1860 continue; 1861 if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) { 1862 if (cpu == curcpu) 1863 drain_local_stock(&stock->work); 1864 else 1865 schedule_work_on(cpu, &stock->work); 1866 } 1867 } 1868 put_cpu(); 1869 put_online_cpus(); 1870 mutex_unlock(&percpu_charge_mutex); 1871 } 1872 1873 static int memcg_cpu_hotplug_callback(struct notifier_block *nb, 1874 unsigned long action, 1875 void *hcpu) 1876 { 1877 int cpu = (unsigned long)hcpu; 1878 struct memcg_stock_pcp *stock; 1879 1880 if (action == CPU_ONLINE) 1881 return NOTIFY_OK; 1882 1883 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN) 1884 return NOTIFY_OK; 1885 1886 stock = &per_cpu(memcg_stock, cpu); 1887 drain_stock(stock); 1888 return NOTIFY_OK; 1889 } 1890 1891 static void reclaim_high(struct mem_cgroup *memcg, 1892 unsigned int nr_pages, 1893 gfp_t gfp_mask) 1894 { 1895 do { 1896 if (page_counter_read(&memcg->memory) <= memcg->high) 1897 continue; 1898 mem_cgroup_events(memcg, MEMCG_HIGH, 1); 1899 try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true); 1900 } while ((memcg = parent_mem_cgroup(memcg))); 1901 } 1902 1903 static void high_work_func(struct work_struct *work) 1904 { 1905 struct mem_cgroup *memcg; 1906 1907 memcg = container_of(work, struct mem_cgroup, high_work); 1908 reclaim_high(memcg, CHARGE_BATCH, GFP_KERNEL); 1909 } 1910 1911 /* 1912 * Scheduled by try_charge() to be executed from the userland return path 1913 * and reclaims memory over the high limit. 1914 */ 1915 void mem_cgroup_handle_over_high(void) 1916 { 1917 unsigned int nr_pages = current->memcg_nr_pages_over_high; 1918 struct mem_cgroup *memcg; 1919 1920 if (likely(!nr_pages)) 1921 return; 1922 1923 memcg = get_mem_cgroup_from_mm(current->mm); 1924 reclaim_high(memcg, nr_pages, GFP_KERNEL); 1925 css_put(&memcg->css); 1926 current->memcg_nr_pages_over_high = 0; 1927 } 1928 1929 static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, 1930 unsigned int nr_pages) 1931 { 1932 unsigned int batch = max(CHARGE_BATCH, nr_pages); 1933 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES; 1934 struct mem_cgroup *mem_over_limit; 1935 struct page_counter *counter; 1936 unsigned long nr_reclaimed; 1937 bool may_swap = true; 1938 bool drained = false; 1939 1940 if (mem_cgroup_is_root(memcg)) 1941 return 0; 1942 retry: 1943 if (consume_stock(memcg, nr_pages)) 1944 return 0; 1945 1946 if (!do_memsw_account() || 1947 page_counter_try_charge(&memcg->memsw, batch, &counter)) { 1948 if (page_counter_try_charge(&memcg->memory, batch, &counter)) 1949 goto done_restock; 1950 if (do_memsw_account()) 1951 page_counter_uncharge(&memcg->memsw, batch); 1952 mem_over_limit = mem_cgroup_from_counter(counter, memory); 1953 } else { 1954 mem_over_limit = mem_cgroup_from_counter(counter, memsw); 1955 may_swap = false; 1956 } 1957 1958 if (batch > nr_pages) { 1959 batch = nr_pages; 1960 goto retry; 1961 } 1962 1963 /* 1964 * Unlike in global OOM situations, memcg is not in a physical 1965 * memory shortage. Allow dying and OOM-killed tasks to 1966 * bypass the last charges so that they can exit quickly and 1967 * free their memory. 1968 */ 1969 if (unlikely(test_thread_flag(TIF_MEMDIE) || 1970 fatal_signal_pending(current) || 1971 current->flags & PF_EXITING)) 1972 goto force; 1973 1974 if (unlikely(task_in_memcg_oom(current))) 1975 goto nomem; 1976 1977 if (!gfpflags_allow_blocking(gfp_mask)) 1978 goto nomem; 1979 1980 mem_cgroup_events(mem_over_limit, MEMCG_MAX, 1); 1981 1982 nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages, 1983 gfp_mask, may_swap); 1984 1985 if (mem_cgroup_margin(mem_over_limit) >= nr_pages) 1986 goto retry; 1987 1988 if (!drained) { 1989 drain_all_stock(mem_over_limit); 1990 drained = true; 1991 goto retry; 1992 } 1993 1994 if (gfp_mask & __GFP_NORETRY) 1995 goto nomem; 1996 /* 1997 * Even though the limit is exceeded at this point, reclaim 1998 * may have been able to free some pages. Retry the charge 1999 * before killing the task. 2000 * 2001 * Only for regular pages, though: huge pages are rather 2002 * unlikely to succeed so close to the limit, and we fall back 2003 * to regular pages anyway in case of failure. 2004 */ 2005 if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER)) 2006 goto retry; 2007 /* 2008 * At task move, charge accounts can be doubly counted. So, it's 2009 * better to wait until the end of task_move if something is going on. 2010 */ 2011 if (mem_cgroup_wait_acct_move(mem_over_limit)) 2012 goto retry; 2013 2014 if (nr_retries--) 2015 goto retry; 2016 2017 if (gfp_mask & __GFP_NOFAIL) 2018 goto force; 2019 2020 if (fatal_signal_pending(current)) 2021 goto force; 2022 2023 mem_cgroup_events(mem_over_limit, MEMCG_OOM, 1); 2024 2025 mem_cgroup_oom(mem_over_limit, gfp_mask, 2026 get_order(nr_pages * PAGE_SIZE)); 2027 nomem: 2028 if (!(gfp_mask & __GFP_NOFAIL)) 2029 return -ENOMEM; 2030 force: 2031 /* 2032 * The allocation either can't fail or will lead to more memory 2033 * being freed very soon. Allow memory usage go over the limit 2034 * temporarily by force charging it. 2035 */ 2036 page_counter_charge(&memcg->memory, nr_pages); 2037 if (do_memsw_account()) 2038 page_counter_charge(&memcg->memsw, nr_pages); 2039 css_get_many(&memcg->css, nr_pages); 2040 2041 return 0; 2042 2043 done_restock: 2044 css_get_many(&memcg->css, batch); 2045 if (batch > nr_pages) 2046 refill_stock(memcg, batch - nr_pages); 2047 2048 /* 2049 * If the hierarchy is above the normal consumption range, schedule 2050 * reclaim on returning to userland. We can perform reclaim here 2051 * if __GFP_RECLAIM but let's always punt for simplicity and so that 2052 * GFP_KERNEL can consistently be used during reclaim. @memcg is 2053 * not recorded as it most likely matches current's and won't 2054 * change in the meantime. As high limit is checked again before 2055 * reclaim, the cost of mismatch is negligible. 2056 */ 2057 do { 2058 if (page_counter_read(&memcg->memory) > memcg->high) { 2059 /* Don't bother a random interrupted task */ 2060 if (in_interrupt()) { 2061 schedule_work(&memcg->high_work); 2062 break; 2063 } 2064 current->memcg_nr_pages_over_high += batch; 2065 set_notify_resume(current); 2066 break; 2067 } 2068 } while ((memcg = parent_mem_cgroup(memcg))); 2069 2070 return 0; 2071 } 2072 2073 static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages) 2074 { 2075 if (mem_cgroup_is_root(memcg)) 2076 return; 2077 2078 page_counter_uncharge(&memcg->memory, nr_pages); 2079 if (do_memsw_account()) 2080 page_counter_uncharge(&memcg->memsw, nr_pages); 2081 2082 css_put_many(&memcg->css, nr_pages); 2083 } 2084 2085 static void lock_page_lru(struct page *page, int *isolated) 2086 { 2087 struct zone *zone = page_zone(page); 2088 2089 spin_lock_irq(&zone->lru_lock); 2090 if (PageLRU(page)) { 2091 struct lruvec *lruvec; 2092 2093 lruvec = mem_cgroup_page_lruvec(page, zone); 2094 ClearPageLRU(page); 2095 del_page_from_lru_list(page, lruvec, page_lru(page)); 2096 *isolated = 1; 2097 } else 2098 *isolated = 0; 2099 } 2100 2101 static void unlock_page_lru(struct page *page, int isolated) 2102 { 2103 struct zone *zone = page_zone(page); 2104 2105 if (isolated) { 2106 struct lruvec *lruvec; 2107 2108 lruvec = mem_cgroup_page_lruvec(page, zone); 2109 VM_BUG_ON_PAGE(PageLRU(page), page); 2110 SetPageLRU(page); 2111 add_page_to_lru_list(page, lruvec, page_lru(page)); 2112 } 2113 spin_unlock_irq(&zone->lru_lock); 2114 } 2115 2116 static void commit_charge(struct page *page, struct mem_cgroup *memcg, 2117 bool lrucare) 2118 { 2119 int isolated; 2120 2121 VM_BUG_ON_PAGE(page->mem_cgroup, page); 2122 2123 /* 2124 * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page 2125 * may already be on some other mem_cgroup's LRU. Take care of it. 2126 */ 2127 if (lrucare) 2128 lock_page_lru(page, &isolated); 2129 2130 /* 2131 * Nobody should be changing or seriously looking at 2132 * page->mem_cgroup at this point: 2133 * 2134 * - the page is uncharged 2135 * 2136 * - the page is off-LRU 2137 * 2138 * - an anonymous fault has exclusive page access, except for 2139 * a locked page table 2140 * 2141 * - a page cache insertion, a swapin fault, or a migration 2142 * have the page locked 2143 */ 2144 page->mem_cgroup = memcg; 2145 2146 if (lrucare) 2147 unlock_page_lru(page, isolated); 2148 } 2149 2150 #ifndef CONFIG_SLOB 2151 static int memcg_alloc_cache_id(void) 2152 { 2153 int id, size; 2154 int err; 2155 2156 id = ida_simple_get(&memcg_cache_ida, 2157 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL); 2158 if (id < 0) 2159 return id; 2160 2161 if (id < memcg_nr_cache_ids) 2162 return id; 2163 2164 /* 2165 * There's no space for the new id in memcg_caches arrays, 2166 * so we have to grow them. 2167 */ 2168 down_write(&memcg_cache_ids_sem); 2169 2170 size = 2 * (id + 1); 2171 if (size < MEMCG_CACHES_MIN_SIZE) 2172 size = MEMCG_CACHES_MIN_SIZE; 2173 else if (size > MEMCG_CACHES_MAX_SIZE) 2174 size = MEMCG_CACHES_MAX_SIZE; 2175 2176 err = memcg_update_all_caches(size); 2177 if (!err) 2178 err = memcg_update_all_list_lrus(size); 2179 if (!err) 2180 memcg_nr_cache_ids = size; 2181 2182 up_write(&memcg_cache_ids_sem); 2183 2184 if (err) { 2185 ida_simple_remove(&memcg_cache_ida, id); 2186 return err; 2187 } 2188 return id; 2189 } 2190 2191 static void memcg_free_cache_id(int id) 2192 { 2193 ida_simple_remove(&memcg_cache_ida, id); 2194 } 2195 2196 struct memcg_kmem_cache_create_work { 2197 struct mem_cgroup *memcg; 2198 struct kmem_cache *cachep; 2199 struct work_struct work; 2200 }; 2201 2202 static void memcg_kmem_cache_create_func(struct work_struct *w) 2203 { 2204 struct memcg_kmem_cache_create_work *cw = 2205 container_of(w, struct memcg_kmem_cache_create_work, work); 2206 struct mem_cgroup *memcg = cw->memcg; 2207 struct kmem_cache *cachep = cw->cachep; 2208 2209 memcg_create_kmem_cache(memcg, cachep); 2210 2211 css_put(&memcg->css); 2212 kfree(cw); 2213 } 2214 2215 /* 2216 * Enqueue the creation of a per-memcg kmem_cache. 2217 */ 2218 static void __memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg, 2219 struct kmem_cache *cachep) 2220 { 2221 struct memcg_kmem_cache_create_work *cw; 2222 2223 cw = kmalloc(sizeof(*cw), GFP_NOWAIT); 2224 if (!cw) 2225 return; 2226 2227 css_get(&memcg->css); 2228 2229 cw->memcg = memcg; 2230 cw->cachep = cachep; 2231 INIT_WORK(&cw->work, memcg_kmem_cache_create_func); 2232 2233 schedule_work(&cw->work); 2234 } 2235 2236 static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg, 2237 struct kmem_cache *cachep) 2238 { 2239 /* 2240 * We need to stop accounting when we kmalloc, because if the 2241 * corresponding kmalloc cache is not yet created, the first allocation 2242 * in __memcg_schedule_kmem_cache_create will recurse. 2243 * 2244 * However, it is better to enclose the whole function. Depending on 2245 * the debugging options enabled, INIT_WORK(), for instance, can 2246 * trigger an allocation. This too, will make us recurse. Because at 2247 * this point we can't allow ourselves back into memcg_kmem_get_cache, 2248 * the safest choice is to do it like this, wrapping the whole function. 2249 */ 2250 current->memcg_kmem_skip_account = 1; 2251 __memcg_schedule_kmem_cache_create(memcg, cachep); 2252 current->memcg_kmem_skip_account = 0; 2253 } 2254 2255 /* 2256 * Return the kmem_cache we're supposed to use for a slab allocation. 2257 * We try to use the current memcg's version of the cache. 2258 * 2259 * If the cache does not exist yet, if we are the first user of it, 2260 * we either create it immediately, if possible, or create it asynchronously 2261 * in a workqueue. 2262 * In the latter case, we will let the current allocation go through with 2263 * the original cache. 2264 * 2265 * Can't be called in interrupt context or from kernel threads. 2266 * This function needs to be called with rcu_read_lock() held. 2267 */ 2268 struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp) 2269 { 2270 struct mem_cgroup *memcg; 2271 struct kmem_cache *memcg_cachep; 2272 int kmemcg_id; 2273 2274 VM_BUG_ON(!is_root_cache(cachep)); 2275 2276 if (cachep->flags & SLAB_ACCOUNT) 2277 gfp |= __GFP_ACCOUNT; 2278 2279 if (!(gfp & __GFP_ACCOUNT)) 2280 return cachep; 2281 2282 if (current->memcg_kmem_skip_account) 2283 return cachep; 2284 2285 memcg = get_mem_cgroup_from_mm(current->mm); 2286 kmemcg_id = READ_ONCE(memcg->kmemcg_id); 2287 if (kmemcg_id < 0) 2288 goto out; 2289 2290 memcg_cachep = cache_from_memcg_idx(cachep, kmemcg_id); 2291 if (likely(memcg_cachep)) 2292 return memcg_cachep; 2293 2294 /* 2295 * If we are in a safe context (can wait, and not in interrupt 2296 * context), we could be be predictable and return right away. 2297 * This would guarantee that the allocation being performed 2298 * already belongs in the new cache. 2299 * 2300 * However, there are some clashes that can arrive from locking. 2301 * For instance, because we acquire the slab_mutex while doing 2302 * memcg_create_kmem_cache, this means no further allocation 2303 * could happen with the slab_mutex held. So it's better to 2304 * defer everything. 2305 */ 2306 memcg_schedule_kmem_cache_create(memcg, cachep); 2307 out: 2308 css_put(&memcg->css); 2309 return cachep; 2310 } 2311 2312 void __memcg_kmem_put_cache(struct kmem_cache *cachep) 2313 { 2314 if (!is_root_cache(cachep)) 2315 css_put(&cachep->memcg_params.memcg->css); 2316 } 2317 2318 int __memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order, 2319 struct mem_cgroup *memcg) 2320 { 2321 unsigned int nr_pages = 1 << order; 2322 struct page_counter *counter; 2323 int ret; 2324 2325 ret = try_charge(memcg, gfp, nr_pages); 2326 if (ret) 2327 return ret; 2328 2329 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && 2330 !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) { 2331 cancel_charge(memcg, nr_pages); 2332 return -ENOMEM; 2333 } 2334 2335 page->mem_cgroup = memcg; 2336 2337 return 0; 2338 } 2339 2340 int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order) 2341 { 2342 struct mem_cgroup *memcg; 2343 int ret = 0; 2344 2345 memcg = get_mem_cgroup_from_mm(current->mm); 2346 if (!mem_cgroup_is_root(memcg)) 2347 ret = __memcg_kmem_charge_memcg(page, gfp, order, memcg); 2348 css_put(&memcg->css); 2349 return ret; 2350 } 2351 2352 void __memcg_kmem_uncharge(struct page *page, int order) 2353 { 2354 struct mem_cgroup *memcg = page->mem_cgroup; 2355 unsigned int nr_pages = 1 << order; 2356 2357 if (!memcg) 2358 return; 2359 2360 VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page); 2361 2362 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) 2363 page_counter_uncharge(&memcg->kmem, nr_pages); 2364 2365 page_counter_uncharge(&memcg->memory, nr_pages); 2366 if (do_memsw_account()) 2367 page_counter_uncharge(&memcg->memsw, nr_pages); 2368 2369 page->mem_cgroup = NULL; 2370 css_put_many(&memcg->css, nr_pages); 2371 } 2372 #endif /* !CONFIG_SLOB */ 2373 2374 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2375 2376 /* 2377 * Because tail pages are not marked as "used", set it. We're under 2378 * zone->lru_lock and migration entries setup in all page mappings. 2379 */ 2380 void mem_cgroup_split_huge_fixup(struct page *head) 2381 { 2382 int i; 2383 2384 if (mem_cgroup_disabled()) 2385 return; 2386 2387 for (i = 1; i < HPAGE_PMD_NR; i++) 2388 head[i].mem_cgroup = head->mem_cgroup; 2389 2390 __this_cpu_sub(head->mem_cgroup->stat->count[MEM_CGROUP_STAT_RSS_HUGE], 2391 HPAGE_PMD_NR); 2392 } 2393 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 2394 2395 #ifdef CONFIG_MEMCG_SWAP 2396 static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg, 2397 bool charge) 2398 { 2399 int val = (charge) ? 1 : -1; 2400 this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAP], val); 2401 } 2402 2403 /** 2404 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record. 2405 * @entry: swap entry to be moved 2406 * @from: mem_cgroup which the entry is moved from 2407 * @to: mem_cgroup which the entry is moved to 2408 * 2409 * It succeeds only when the swap_cgroup's record for this entry is the same 2410 * as the mem_cgroup's id of @from. 2411 * 2412 * Returns 0 on success, -EINVAL on failure. 2413 * 2414 * The caller must have charged to @to, IOW, called page_counter_charge() about 2415 * both res and memsw, and called css_get(). 2416 */ 2417 static int mem_cgroup_move_swap_account(swp_entry_t entry, 2418 struct mem_cgroup *from, struct mem_cgroup *to) 2419 { 2420 unsigned short old_id, new_id; 2421 2422 old_id = mem_cgroup_id(from); 2423 new_id = mem_cgroup_id(to); 2424 2425 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) { 2426 mem_cgroup_swap_statistics(from, false); 2427 mem_cgroup_swap_statistics(to, true); 2428 return 0; 2429 } 2430 return -EINVAL; 2431 } 2432 #else 2433 static inline int mem_cgroup_move_swap_account(swp_entry_t entry, 2434 struct mem_cgroup *from, struct mem_cgroup *to) 2435 { 2436 return -EINVAL; 2437 } 2438 #endif 2439 2440 static DEFINE_MUTEX(memcg_limit_mutex); 2441 2442 static int mem_cgroup_resize_limit(struct mem_cgroup *memcg, 2443 unsigned long limit) 2444 { 2445 unsigned long curusage; 2446 unsigned long oldusage; 2447 bool enlarge = false; 2448 int retry_count; 2449 int ret; 2450 2451 /* 2452 * For keeping hierarchical_reclaim simple, how long we should retry 2453 * is depends on callers. We set our retry-count to be function 2454 * of # of children which we should visit in this loop. 2455 */ 2456 retry_count = MEM_CGROUP_RECLAIM_RETRIES * 2457 mem_cgroup_count_children(memcg); 2458 2459 oldusage = page_counter_read(&memcg->memory); 2460 2461 do { 2462 if (signal_pending(current)) { 2463 ret = -EINTR; 2464 break; 2465 } 2466 2467 mutex_lock(&memcg_limit_mutex); 2468 if (limit > memcg->memsw.limit) { 2469 mutex_unlock(&memcg_limit_mutex); 2470 ret = -EINVAL; 2471 break; 2472 } 2473 if (limit > memcg->memory.limit) 2474 enlarge = true; 2475 ret = page_counter_limit(&memcg->memory, limit); 2476 mutex_unlock(&memcg_limit_mutex); 2477 2478 if (!ret) 2479 break; 2480 2481 try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, true); 2482 2483 curusage = page_counter_read(&memcg->memory); 2484 /* Usage is reduced ? */ 2485 if (curusage >= oldusage) 2486 retry_count--; 2487 else 2488 oldusage = curusage; 2489 } while (retry_count); 2490 2491 if (!ret && enlarge) 2492 memcg_oom_recover(memcg); 2493 2494 return ret; 2495 } 2496 2497 static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg, 2498 unsigned long limit) 2499 { 2500 unsigned long curusage; 2501 unsigned long oldusage; 2502 bool enlarge = false; 2503 int retry_count; 2504 int ret; 2505 2506 /* see mem_cgroup_resize_res_limit */ 2507 retry_count = MEM_CGROUP_RECLAIM_RETRIES * 2508 mem_cgroup_count_children(memcg); 2509 2510 oldusage = page_counter_read(&memcg->memsw); 2511 2512 do { 2513 if (signal_pending(current)) { 2514 ret = -EINTR; 2515 break; 2516 } 2517 2518 mutex_lock(&memcg_limit_mutex); 2519 if (limit < memcg->memory.limit) { 2520 mutex_unlock(&memcg_limit_mutex); 2521 ret = -EINVAL; 2522 break; 2523 } 2524 if (limit > memcg->memsw.limit) 2525 enlarge = true; 2526 ret = page_counter_limit(&memcg->memsw, limit); 2527 mutex_unlock(&memcg_limit_mutex); 2528 2529 if (!ret) 2530 break; 2531 2532 try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, false); 2533 2534 curusage = page_counter_read(&memcg->memsw); 2535 /* Usage is reduced ? */ 2536 if (curusage >= oldusage) 2537 retry_count--; 2538 else 2539 oldusage = curusage; 2540 } while (retry_count); 2541 2542 if (!ret && enlarge) 2543 memcg_oom_recover(memcg); 2544 2545 return ret; 2546 } 2547 2548 unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order, 2549 gfp_t gfp_mask, 2550 unsigned long *total_scanned) 2551 { 2552 unsigned long nr_reclaimed = 0; 2553 struct mem_cgroup_per_zone *mz, *next_mz = NULL; 2554 unsigned long reclaimed; 2555 int loop = 0; 2556 struct mem_cgroup_tree_per_zone *mctz; 2557 unsigned long excess; 2558 unsigned long nr_scanned; 2559 2560 if (order > 0) 2561 return 0; 2562 2563 mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone)); 2564 /* 2565 * This loop can run a while, specially if mem_cgroup's continuously 2566 * keep exceeding their soft limit and putting the system under 2567 * pressure 2568 */ 2569 do { 2570 if (next_mz) 2571 mz = next_mz; 2572 else 2573 mz = mem_cgroup_largest_soft_limit_node(mctz); 2574 if (!mz) 2575 break; 2576 2577 nr_scanned = 0; 2578 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, zone, 2579 gfp_mask, &nr_scanned); 2580 nr_reclaimed += reclaimed; 2581 *total_scanned += nr_scanned; 2582 spin_lock_irq(&mctz->lock); 2583 __mem_cgroup_remove_exceeded(mz, mctz); 2584 2585 /* 2586 * If we failed to reclaim anything from this memory cgroup 2587 * it is time to move on to the next cgroup 2588 */ 2589 next_mz = NULL; 2590 if (!reclaimed) 2591 next_mz = __mem_cgroup_largest_soft_limit_node(mctz); 2592 2593 excess = soft_limit_excess(mz->memcg); 2594 /* 2595 * One school of thought says that we should not add 2596 * back the node to the tree if reclaim returns 0. 2597 * But our reclaim could return 0, simply because due 2598 * to priority we are exposing a smaller subset of 2599 * memory to reclaim from. Consider this as a longer 2600 * term TODO. 2601 */ 2602 /* If excess == 0, no tree ops */ 2603 __mem_cgroup_insert_exceeded(mz, mctz, excess); 2604 spin_unlock_irq(&mctz->lock); 2605 css_put(&mz->memcg->css); 2606 loop++; 2607 /* 2608 * Could not reclaim anything and there are no more 2609 * mem cgroups to try or we seem to be looping without 2610 * reclaiming anything. 2611 */ 2612 if (!nr_reclaimed && 2613 (next_mz == NULL || 2614 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS)) 2615 break; 2616 } while (!nr_reclaimed); 2617 if (next_mz) 2618 css_put(&next_mz->memcg->css); 2619 return nr_reclaimed; 2620 } 2621 2622 /* 2623 * Test whether @memcg has children, dead or alive. Note that this 2624 * function doesn't care whether @memcg has use_hierarchy enabled and 2625 * returns %true if there are child csses according to the cgroup 2626 * hierarchy. Testing use_hierarchy is the caller's responsiblity. 2627 */ 2628 static inline bool memcg_has_children(struct mem_cgroup *memcg) 2629 { 2630 bool ret; 2631 2632 rcu_read_lock(); 2633 ret = css_next_child(NULL, &memcg->css); 2634 rcu_read_unlock(); 2635 return ret; 2636 } 2637 2638 /* 2639 * Reclaims as many pages from the given memcg as possible and moves 2640 * the rest to the parent. 2641 * 2642 * Caller is responsible for holding css reference for memcg. 2643 */ 2644 static int mem_cgroup_force_empty(struct mem_cgroup *memcg) 2645 { 2646 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES; 2647 2648 /* we call try-to-free pages for make this cgroup empty */ 2649 lru_add_drain_all(); 2650 /* try to free all pages in this cgroup */ 2651 while (nr_retries && page_counter_read(&memcg->memory)) { 2652 int progress; 2653 2654 if (signal_pending(current)) 2655 return -EINTR; 2656 2657 progress = try_to_free_mem_cgroup_pages(memcg, 1, 2658 GFP_KERNEL, true); 2659 if (!progress) { 2660 nr_retries--; 2661 /* maybe some writeback is necessary */ 2662 congestion_wait(BLK_RW_ASYNC, HZ/10); 2663 } 2664 2665 } 2666 2667 return 0; 2668 } 2669 2670 static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of, 2671 char *buf, size_t nbytes, 2672 loff_t off) 2673 { 2674 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 2675 2676 if (mem_cgroup_is_root(memcg)) 2677 return -EINVAL; 2678 return mem_cgroup_force_empty(memcg) ?: nbytes; 2679 } 2680 2681 static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css, 2682 struct cftype *cft) 2683 { 2684 return mem_cgroup_from_css(css)->use_hierarchy; 2685 } 2686 2687 static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css, 2688 struct cftype *cft, u64 val) 2689 { 2690 int retval = 0; 2691 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 2692 struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent); 2693 2694 if (memcg->use_hierarchy == val) 2695 return 0; 2696 2697 /* 2698 * If parent's use_hierarchy is set, we can't make any modifications 2699 * in the child subtrees. If it is unset, then the change can 2700 * occur, provided the current cgroup has no children. 2701 * 2702 * For the root cgroup, parent_mem is NULL, we allow value to be 2703 * set if there are no children. 2704 */ 2705 if ((!parent_memcg || !parent_memcg->use_hierarchy) && 2706 (val == 1 || val == 0)) { 2707 if (!memcg_has_children(memcg)) 2708 memcg->use_hierarchy = val; 2709 else 2710 retval = -EBUSY; 2711 } else 2712 retval = -EINVAL; 2713 2714 return retval; 2715 } 2716 2717 static void tree_stat(struct mem_cgroup *memcg, unsigned long *stat) 2718 { 2719 struct mem_cgroup *iter; 2720 int i; 2721 2722 memset(stat, 0, sizeof(*stat) * MEMCG_NR_STAT); 2723 2724 for_each_mem_cgroup_tree(iter, memcg) { 2725 for (i = 0; i < MEMCG_NR_STAT; i++) 2726 stat[i] += mem_cgroup_read_stat(iter, i); 2727 } 2728 } 2729 2730 static void tree_events(struct mem_cgroup *memcg, unsigned long *events) 2731 { 2732 struct mem_cgroup *iter; 2733 int i; 2734 2735 memset(events, 0, sizeof(*events) * MEMCG_NR_EVENTS); 2736 2737 for_each_mem_cgroup_tree(iter, memcg) { 2738 for (i = 0; i < MEMCG_NR_EVENTS; i++) 2739 events[i] += mem_cgroup_read_events(iter, i); 2740 } 2741 } 2742 2743 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap) 2744 { 2745 unsigned long val = 0; 2746 2747 if (mem_cgroup_is_root(memcg)) { 2748 struct mem_cgroup *iter; 2749 2750 for_each_mem_cgroup_tree(iter, memcg) { 2751 val += mem_cgroup_read_stat(iter, 2752 MEM_CGROUP_STAT_CACHE); 2753 val += mem_cgroup_read_stat(iter, 2754 MEM_CGROUP_STAT_RSS); 2755 if (swap) 2756 val += mem_cgroup_read_stat(iter, 2757 MEM_CGROUP_STAT_SWAP); 2758 } 2759 } else { 2760 if (!swap) 2761 val = page_counter_read(&memcg->memory); 2762 else 2763 val = page_counter_read(&memcg->memsw); 2764 } 2765 return val; 2766 } 2767 2768 enum { 2769 RES_USAGE, 2770 RES_LIMIT, 2771 RES_MAX_USAGE, 2772 RES_FAILCNT, 2773 RES_SOFT_LIMIT, 2774 }; 2775 2776 static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css, 2777 struct cftype *cft) 2778 { 2779 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 2780 struct page_counter *counter; 2781 2782 switch (MEMFILE_TYPE(cft->private)) { 2783 case _MEM: 2784 counter = &memcg->memory; 2785 break; 2786 case _MEMSWAP: 2787 counter = &memcg->memsw; 2788 break; 2789 case _KMEM: 2790 counter = &memcg->kmem; 2791 break; 2792 case _TCP: 2793 counter = &memcg->tcpmem; 2794 break; 2795 default: 2796 BUG(); 2797 } 2798 2799 switch (MEMFILE_ATTR(cft->private)) { 2800 case RES_USAGE: 2801 if (counter == &memcg->memory) 2802 return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE; 2803 if (counter == &memcg->memsw) 2804 return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE; 2805 return (u64)page_counter_read(counter) * PAGE_SIZE; 2806 case RES_LIMIT: 2807 return (u64)counter->limit * PAGE_SIZE; 2808 case RES_MAX_USAGE: 2809 return (u64)counter->watermark * PAGE_SIZE; 2810 case RES_FAILCNT: 2811 return counter->failcnt; 2812 case RES_SOFT_LIMIT: 2813 return (u64)memcg->soft_limit * PAGE_SIZE; 2814 default: 2815 BUG(); 2816 } 2817 } 2818 2819 #ifndef CONFIG_SLOB 2820 static int memcg_online_kmem(struct mem_cgroup *memcg) 2821 { 2822 int memcg_id; 2823 2824 if (cgroup_memory_nokmem) 2825 return 0; 2826 2827 BUG_ON(memcg->kmemcg_id >= 0); 2828 BUG_ON(memcg->kmem_state); 2829 2830 memcg_id = memcg_alloc_cache_id(); 2831 if (memcg_id < 0) 2832 return memcg_id; 2833 2834 static_branch_inc(&memcg_kmem_enabled_key); 2835 /* 2836 * A memory cgroup is considered kmem-online as soon as it gets 2837 * kmemcg_id. Setting the id after enabling static branching will 2838 * guarantee no one starts accounting before all call sites are 2839 * patched. 2840 */ 2841 memcg->kmemcg_id = memcg_id; 2842 memcg->kmem_state = KMEM_ONLINE; 2843 2844 return 0; 2845 } 2846 2847 static void memcg_offline_kmem(struct mem_cgroup *memcg) 2848 { 2849 struct cgroup_subsys_state *css; 2850 struct mem_cgroup *parent, *child; 2851 int kmemcg_id; 2852 2853 if (memcg->kmem_state != KMEM_ONLINE) 2854 return; 2855 /* 2856 * Clear the online state before clearing memcg_caches array 2857 * entries. The slab_mutex in memcg_deactivate_kmem_caches() 2858 * guarantees that no cache will be created for this cgroup 2859 * after we are done (see memcg_create_kmem_cache()). 2860 */ 2861 memcg->kmem_state = KMEM_ALLOCATED; 2862 2863 memcg_deactivate_kmem_caches(memcg); 2864 2865 kmemcg_id = memcg->kmemcg_id; 2866 BUG_ON(kmemcg_id < 0); 2867 2868 parent = parent_mem_cgroup(memcg); 2869 if (!parent) 2870 parent = root_mem_cgroup; 2871 2872 /* 2873 * Change kmemcg_id of this cgroup and all its descendants to the 2874 * parent's id, and then move all entries from this cgroup's list_lrus 2875 * to ones of the parent. After we have finished, all list_lrus 2876 * corresponding to this cgroup are guaranteed to remain empty. The 2877 * ordering is imposed by list_lru_node->lock taken by 2878 * memcg_drain_all_list_lrus(). 2879 */ 2880 css_for_each_descendant_pre(css, &memcg->css) { 2881 child = mem_cgroup_from_css(css); 2882 BUG_ON(child->kmemcg_id != kmemcg_id); 2883 child->kmemcg_id = parent->kmemcg_id; 2884 if (!memcg->use_hierarchy) 2885 break; 2886 } 2887 memcg_drain_all_list_lrus(kmemcg_id, parent->kmemcg_id); 2888 2889 memcg_free_cache_id(kmemcg_id); 2890 } 2891 2892 static void memcg_free_kmem(struct mem_cgroup *memcg) 2893 { 2894 /* css_alloc() failed, offlining didn't happen */ 2895 if (unlikely(memcg->kmem_state == KMEM_ONLINE)) 2896 memcg_offline_kmem(memcg); 2897 2898 if (memcg->kmem_state == KMEM_ALLOCATED) { 2899 memcg_destroy_kmem_caches(memcg); 2900 static_branch_dec(&memcg_kmem_enabled_key); 2901 WARN_ON(page_counter_read(&memcg->kmem)); 2902 } 2903 } 2904 #else 2905 static int memcg_online_kmem(struct mem_cgroup *memcg) 2906 { 2907 return 0; 2908 } 2909 static void memcg_offline_kmem(struct mem_cgroup *memcg) 2910 { 2911 } 2912 static void memcg_free_kmem(struct mem_cgroup *memcg) 2913 { 2914 } 2915 #endif /* !CONFIG_SLOB */ 2916 2917 static int memcg_update_kmem_limit(struct mem_cgroup *memcg, 2918 unsigned long limit) 2919 { 2920 int ret; 2921 2922 mutex_lock(&memcg_limit_mutex); 2923 ret = page_counter_limit(&memcg->kmem, limit); 2924 mutex_unlock(&memcg_limit_mutex); 2925 return ret; 2926 } 2927 2928 static int memcg_update_tcp_limit(struct mem_cgroup *memcg, unsigned long limit) 2929 { 2930 int ret; 2931 2932 mutex_lock(&memcg_limit_mutex); 2933 2934 ret = page_counter_limit(&memcg->tcpmem, limit); 2935 if (ret) 2936 goto out; 2937 2938 if (!memcg->tcpmem_active) { 2939 /* 2940 * The active flag needs to be written after the static_key 2941 * update. This is what guarantees that the socket activation 2942 * function is the last one to run. See sock_update_memcg() for 2943 * details, and note that we don't mark any socket as belonging 2944 * to this memcg until that flag is up. 2945 * 2946 * We need to do this, because static_keys will span multiple 2947 * sites, but we can't control their order. If we mark a socket 2948 * as accounted, but the accounting functions are not patched in 2949 * yet, we'll lose accounting. 2950 * 2951 * We never race with the readers in sock_update_memcg(), 2952 * because when this value change, the code to process it is not 2953 * patched in yet. 2954 */ 2955 static_branch_inc(&memcg_sockets_enabled_key); 2956 memcg->tcpmem_active = true; 2957 } 2958 out: 2959 mutex_unlock(&memcg_limit_mutex); 2960 return ret; 2961 } 2962 2963 /* 2964 * The user of this function is... 2965 * RES_LIMIT. 2966 */ 2967 static ssize_t mem_cgroup_write(struct kernfs_open_file *of, 2968 char *buf, size_t nbytes, loff_t off) 2969 { 2970 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 2971 unsigned long nr_pages; 2972 int ret; 2973 2974 buf = strstrip(buf); 2975 ret = page_counter_memparse(buf, "-1", &nr_pages); 2976 if (ret) 2977 return ret; 2978 2979 switch (MEMFILE_ATTR(of_cft(of)->private)) { 2980 case RES_LIMIT: 2981 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */ 2982 ret = -EINVAL; 2983 break; 2984 } 2985 switch (MEMFILE_TYPE(of_cft(of)->private)) { 2986 case _MEM: 2987 ret = mem_cgroup_resize_limit(memcg, nr_pages); 2988 break; 2989 case _MEMSWAP: 2990 ret = mem_cgroup_resize_memsw_limit(memcg, nr_pages); 2991 break; 2992 case _KMEM: 2993 ret = memcg_update_kmem_limit(memcg, nr_pages); 2994 break; 2995 case _TCP: 2996 ret = memcg_update_tcp_limit(memcg, nr_pages); 2997 break; 2998 } 2999 break; 3000 case RES_SOFT_LIMIT: 3001 memcg->soft_limit = nr_pages; 3002 ret = 0; 3003 break; 3004 } 3005 return ret ?: nbytes; 3006 } 3007 3008 static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf, 3009 size_t nbytes, loff_t off) 3010 { 3011 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 3012 struct page_counter *counter; 3013 3014 switch (MEMFILE_TYPE(of_cft(of)->private)) { 3015 case _MEM: 3016 counter = &memcg->memory; 3017 break; 3018 case _MEMSWAP: 3019 counter = &memcg->memsw; 3020 break; 3021 case _KMEM: 3022 counter = &memcg->kmem; 3023 break; 3024 case _TCP: 3025 counter = &memcg->tcpmem; 3026 break; 3027 default: 3028 BUG(); 3029 } 3030 3031 switch (MEMFILE_ATTR(of_cft(of)->private)) { 3032 case RES_MAX_USAGE: 3033 page_counter_reset_watermark(counter); 3034 break; 3035 case RES_FAILCNT: 3036 counter->failcnt = 0; 3037 break; 3038 default: 3039 BUG(); 3040 } 3041 3042 return nbytes; 3043 } 3044 3045 static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css, 3046 struct cftype *cft) 3047 { 3048 return mem_cgroup_from_css(css)->move_charge_at_immigrate; 3049 } 3050 3051 #ifdef CONFIG_MMU 3052 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css, 3053 struct cftype *cft, u64 val) 3054 { 3055 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 3056 3057 if (val & ~MOVE_MASK) 3058 return -EINVAL; 3059 3060 /* 3061 * No kind of locking is needed in here, because ->can_attach() will 3062 * check this value once in the beginning of the process, and then carry 3063 * on with stale data. This means that changes to this value will only 3064 * affect task migrations starting after the change. 3065 */ 3066 memcg->move_charge_at_immigrate = val; 3067 return 0; 3068 } 3069 #else 3070 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css, 3071 struct cftype *cft, u64 val) 3072 { 3073 return -ENOSYS; 3074 } 3075 #endif 3076 3077 #ifdef CONFIG_NUMA 3078 static int memcg_numa_stat_show(struct seq_file *m, void *v) 3079 { 3080 struct numa_stat { 3081 const char *name; 3082 unsigned int lru_mask; 3083 }; 3084 3085 static const struct numa_stat stats[] = { 3086 { "total", LRU_ALL }, 3087 { "file", LRU_ALL_FILE }, 3088 { "anon", LRU_ALL_ANON }, 3089 { "unevictable", BIT(LRU_UNEVICTABLE) }, 3090 }; 3091 const struct numa_stat *stat; 3092 int nid; 3093 unsigned long nr; 3094 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 3095 3096 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) { 3097 nr = mem_cgroup_nr_lru_pages(memcg, stat->lru_mask); 3098 seq_printf(m, "%s=%lu", stat->name, nr); 3099 for_each_node_state(nid, N_MEMORY) { 3100 nr = mem_cgroup_node_nr_lru_pages(memcg, nid, 3101 stat->lru_mask); 3102 seq_printf(m, " N%d=%lu", nid, nr); 3103 } 3104 seq_putc(m, '\n'); 3105 } 3106 3107 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) { 3108 struct mem_cgroup *iter; 3109 3110 nr = 0; 3111 for_each_mem_cgroup_tree(iter, memcg) 3112 nr += mem_cgroup_nr_lru_pages(iter, stat->lru_mask); 3113 seq_printf(m, "hierarchical_%s=%lu", stat->name, nr); 3114 for_each_node_state(nid, N_MEMORY) { 3115 nr = 0; 3116 for_each_mem_cgroup_tree(iter, memcg) 3117 nr += mem_cgroup_node_nr_lru_pages( 3118 iter, nid, stat->lru_mask); 3119 seq_printf(m, " N%d=%lu", nid, nr); 3120 } 3121 seq_putc(m, '\n'); 3122 } 3123 3124 return 0; 3125 } 3126 #endif /* CONFIG_NUMA */ 3127 3128 static int memcg_stat_show(struct seq_file *m, void *v) 3129 { 3130 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 3131 unsigned long memory, memsw; 3132 struct mem_cgroup *mi; 3133 unsigned int i; 3134 3135 BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_stat_names) != 3136 MEM_CGROUP_STAT_NSTATS); 3137 BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_events_names) != 3138 MEM_CGROUP_EVENTS_NSTATS); 3139 BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS); 3140 3141 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) { 3142 if (i == MEM_CGROUP_STAT_SWAP && !do_memsw_account()) 3143 continue; 3144 seq_printf(m, "%s %lu\n", mem_cgroup_stat_names[i], 3145 mem_cgroup_read_stat(memcg, i) * PAGE_SIZE); 3146 } 3147 3148 for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) 3149 seq_printf(m, "%s %lu\n", mem_cgroup_events_names[i], 3150 mem_cgroup_read_events(memcg, i)); 3151 3152 for (i = 0; i < NR_LRU_LISTS; i++) 3153 seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i], 3154 mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE); 3155 3156 /* Hierarchical information */ 3157 memory = memsw = PAGE_COUNTER_MAX; 3158 for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) { 3159 memory = min(memory, mi->memory.limit); 3160 memsw = min(memsw, mi->memsw.limit); 3161 } 3162 seq_printf(m, "hierarchical_memory_limit %llu\n", 3163 (u64)memory * PAGE_SIZE); 3164 if (do_memsw_account()) 3165 seq_printf(m, "hierarchical_memsw_limit %llu\n", 3166 (u64)memsw * PAGE_SIZE); 3167 3168 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) { 3169 unsigned long long val = 0; 3170 3171 if (i == MEM_CGROUP_STAT_SWAP && !do_memsw_account()) 3172 continue; 3173 for_each_mem_cgroup_tree(mi, memcg) 3174 val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE; 3175 seq_printf(m, "total_%s %llu\n", mem_cgroup_stat_names[i], val); 3176 } 3177 3178 for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) { 3179 unsigned long long val = 0; 3180 3181 for_each_mem_cgroup_tree(mi, memcg) 3182 val += mem_cgroup_read_events(mi, i); 3183 seq_printf(m, "total_%s %llu\n", 3184 mem_cgroup_events_names[i], val); 3185 } 3186 3187 for (i = 0; i < NR_LRU_LISTS; i++) { 3188 unsigned long long val = 0; 3189 3190 for_each_mem_cgroup_tree(mi, memcg) 3191 val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE; 3192 seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val); 3193 } 3194 3195 #ifdef CONFIG_DEBUG_VM 3196 { 3197 int nid, zid; 3198 struct mem_cgroup_per_zone *mz; 3199 struct zone_reclaim_stat *rstat; 3200 unsigned long recent_rotated[2] = {0, 0}; 3201 unsigned long recent_scanned[2] = {0, 0}; 3202 3203 for_each_online_node(nid) 3204 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 3205 mz = &memcg->nodeinfo[nid]->zoneinfo[zid]; 3206 rstat = &mz->lruvec.reclaim_stat; 3207 3208 recent_rotated[0] += rstat->recent_rotated[0]; 3209 recent_rotated[1] += rstat->recent_rotated[1]; 3210 recent_scanned[0] += rstat->recent_scanned[0]; 3211 recent_scanned[1] += rstat->recent_scanned[1]; 3212 } 3213 seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]); 3214 seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]); 3215 seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]); 3216 seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]); 3217 } 3218 #endif 3219 3220 return 0; 3221 } 3222 3223 static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css, 3224 struct cftype *cft) 3225 { 3226 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 3227 3228 return mem_cgroup_swappiness(memcg); 3229 } 3230 3231 static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css, 3232 struct cftype *cft, u64 val) 3233 { 3234 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 3235 3236 if (val > 100) 3237 return -EINVAL; 3238 3239 if (css->parent) 3240 memcg->swappiness = val; 3241 else 3242 vm_swappiness = val; 3243 3244 return 0; 3245 } 3246 3247 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap) 3248 { 3249 struct mem_cgroup_threshold_ary *t; 3250 unsigned long usage; 3251 int i; 3252 3253 rcu_read_lock(); 3254 if (!swap) 3255 t = rcu_dereference(memcg->thresholds.primary); 3256 else 3257 t = rcu_dereference(memcg->memsw_thresholds.primary); 3258 3259 if (!t) 3260 goto unlock; 3261 3262 usage = mem_cgroup_usage(memcg, swap); 3263 3264 /* 3265 * current_threshold points to threshold just below or equal to usage. 3266 * If it's not true, a threshold was crossed after last 3267 * call of __mem_cgroup_threshold(). 3268 */ 3269 i = t->current_threshold; 3270 3271 /* 3272 * Iterate backward over array of thresholds starting from 3273 * current_threshold and check if a threshold is crossed. 3274 * If none of thresholds below usage is crossed, we read 3275 * only one element of the array here. 3276 */ 3277 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--) 3278 eventfd_signal(t->entries[i].eventfd, 1); 3279 3280 /* i = current_threshold + 1 */ 3281 i++; 3282 3283 /* 3284 * Iterate forward over array of thresholds starting from 3285 * current_threshold+1 and check if a threshold is crossed. 3286 * If none of thresholds above usage is crossed, we read 3287 * only one element of the array here. 3288 */ 3289 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++) 3290 eventfd_signal(t->entries[i].eventfd, 1); 3291 3292 /* Update current_threshold */ 3293 t->current_threshold = i - 1; 3294 unlock: 3295 rcu_read_unlock(); 3296 } 3297 3298 static void mem_cgroup_threshold(struct mem_cgroup *memcg) 3299 { 3300 while (memcg) { 3301 __mem_cgroup_threshold(memcg, false); 3302 if (do_memsw_account()) 3303 __mem_cgroup_threshold(memcg, true); 3304 3305 memcg = parent_mem_cgroup(memcg); 3306 } 3307 } 3308 3309 static int compare_thresholds(const void *a, const void *b) 3310 { 3311 const struct mem_cgroup_threshold *_a = a; 3312 const struct mem_cgroup_threshold *_b = b; 3313 3314 if (_a->threshold > _b->threshold) 3315 return 1; 3316 3317 if (_a->threshold < _b->threshold) 3318 return -1; 3319 3320 return 0; 3321 } 3322 3323 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg) 3324 { 3325 struct mem_cgroup_eventfd_list *ev; 3326 3327 spin_lock(&memcg_oom_lock); 3328 3329 list_for_each_entry(ev, &memcg->oom_notify, list) 3330 eventfd_signal(ev->eventfd, 1); 3331 3332 spin_unlock(&memcg_oom_lock); 3333 return 0; 3334 } 3335 3336 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg) 3337 { 3338 struct mem_cgroup *iter; 3339 3340 for_each_mem_cgroup_tree(iter, memcg) 3341 mem_cgroup_oom_notify_cb(iter); 3342 } 3343 3344 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg, 3345 struct eventfd_ctx *eventfd, const char *args, enum res_type type) 3346 { 3347 struct mem_cgroup_thresholds *thresholds; 3348 struct mem_cgroup_threshold_ary *new; 3349 unsigned long threshold; 3350 unsigned long usage; 3351 int i, size, ret; 3352 3353 ret = page_counter_memparse(args, "-1", &threshold); 3354 if (ret) 3355 return ret; 3356 3357 mutex_lock(&memcg->thresholds_lock); 3358 3359 if (type == _MEM) { 3360 thresholds = &memcg->thresholds; 3361 usage = mem_cgroup_usage(memcg, false); 3362 } else if (type == _MEMSWAP) { 3363 thresholds = &memcg->memsw_thresholds; 3364 usage = mem_cgroup_usage(memcg, true); 3365 } else 3366 BUG(); 3367 3368 /* Check if a threshold crossed before adding a new one */ 3369 if (thresholds->primary) 3370 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 3371 3372 size = thresholds->primary ? thresholds->primary->size + 1 : 1; 3373 3374 /* Allocate memory for new array of thresholds */ 3375 new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold), 3376 GFP_KERNEL); 3377 if (!new) { 3378 ret = -ENOMEM; 3379 goto unlock; 3380 } 3381 new->size = size; 3382 3383 /* Copy thresholds (if any) to new array */ 3384 if (thresholds->primary) { 3385 memcpy(new->entries, thresholds->primary->entries, (size - 1) * 3386 sizeof(struct mem_cgroup_threshold)); 3387 } 3388 3389 /* Add new threshold */ 3390 new->entries[size - 1].eventfd = eventfd; 3391 new->entries[size - 1].threshold = threshold; 3392 3393 /* Sort thresholds. Registering of new threshold isn't time-critical */ 3394 sort(new->entries, size, sizeof(struct mem_cgroup_threshold), 3395 compare_thresholds, NULL); 3396 3397 /* Find current threshold */ 3398 new->current_threshold = -1; 3399 for (i = 0; i < size; i++) { 3400 if (new->entries[i].threshold <= usage) { 3401 /* 3402 * new->current_threshold will not be used until 3403 * rcu_assign_pointer(), so it's safe to increment 3404 * it here. 3405 */ 3406 ++new->current_threshold; 3407 } else 3408 break; 3409 } 3410 3411 /* Free old spare buffer and save old primary buffer as spare */ 3412 kfree(thresholds->spare); 3413 thresholds->spare = thresholds->primary; 3414 3415 rcu_assign_pointer(thresholds->primary, new); 3416 3417 /* To be sure that nobody uses thresholds */ 3418 synchronize_rcu(); 3419 3420 unlock: 3421 mutex_unlock(&memcg->thresholds_lock); 3422 3423 return ret; 3424 } 3425 3426 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg, 3427 struct eventfd_ctx *eventfd, const char *args) 3428 { 3429 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM); 3430 } 3431 3432 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg, 3433 struct eventfd_ctx *eventfd, const char *args) 3434 { 3435 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP); 3436 } 3437 3438 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, 3439 struct eventfd_ctx *eventfd, enum res_type type) 3440 { 3441 struct mem_cgroup_thresholds *thresholds; 3442 struct mem_cgroup_threshold_ary *new; 3443 unsigned long usage; 3444 int i, j, size; 3445 3446 mutex_lock(&memcg->thresholds_lock); 3447 3448 if (type == _MEM) { 3449 thresholds = &memcg->thresholds; 3450 usage = mem_cgroup_usage(memcg, false); 3451 } else if (type == _MEMSWAP) { 3452 thresholds = &memcg->memsw_thresholds; 3453 usage = mem_cgroup_usage(memcg, true); 3454 } else 3455 BUG(); 3456 3457 if (!thresholds->primary) 3458 goto unlock; 3459 3460 /* Check if a threshold crossed before removing */ 3461 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 3462 3463 /* Calculate new number of threshold */ 3464 size = 0; 3465 for (i = 0; i < thresholds->primary->size; i++) { 3466 if (thresholds->primary->entries[i].eventfd != eventfd) 3467 size++; 3468 } 3469 3470 new = thresholds->spare; 3471 3472 /* Set thresholds array to NULL if we don't have thresholds */ 3473 if (!size) { 3474 kfree(new); 3475 new = NULL; 3476 goto swap_buffers; 3477 } 3478 3479 new->size = size; 3480 3481 /* Copy thresholds and find current threshold */ 3482 new->current_threshold = -1; 3483 for (i = 0, j = 0; i < thresholds->primary->size; i++) { 3484 if (thresholds->primary->entries[i].eventfd == eventfd) 3485 continue; 3486 3487 new->entries[j] = thresholds->primary->entries[i]; 3488 if (new->entries[j].threshold <= usage) { 3489 /* 3490 * new->current_threshold will not be used 3491 * until rcu_assign_pointer(), so it's safe to increment 3492 * it here. 3493 */ 3494 ++new->current_threshold; 3495 } 3496 j++; 3497 } 3498 3499 swap_buffers: 3500 /* Swap primary and spare array */ 3501 thresholds->spare = thresholds->primary; 3502 3503 rcu_assign_pointer(thresholds->primary, new); 3504 3505 /* To be sure that nobody uses thresholds */ 3506 synchronize_rcu(); 3507 3508 /* If all events are unregistered, free the spare array */ 3509 if (!new) { 3510 kfree(thresholds->spare); 3511 thresholds->spare = NULL; 3512 } 3513 unlock: 3514 mutex_unlock(&memcg->thresholds_lock); 3515 } 3516 3517 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, 3518 struct eventfd_ctx *eventfd) 3519 { 3520 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM); 3521 } 3522 3523 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg, 3524 struct eventfd_ctx *eventfd) 3525 { 3526 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP); 3527 } 3528 3529 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg, 3530 struct eventfd_ctx *eventfd, const char *args) 3531 { 3532 struct mem_cgroup_eventfd_list *event; 3533 3534 event = kmalloc(sizeof(*event), GFP_KERNEL); 3535 if (!event) 3536 return -ENOMEM; 3537 3538 spin_lock(&memcg_oom_lock); 3539 3540 event->eventfd = eventfd; 3541 list_add(&event->list, &memcg->oom_notify); 3542 3543 /* already in OOM ? */ 3544 if (memcg->under_oom) 3545 eventfd_signal(eventfd, 1); 3546 spin_unlock(&memcg_oom_lock); 3547 3548 return 0; 3549 } 3550 3551 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg, 3552 struct eventfd_ctx *eventfd) 3553 { 3554 struct mem_cgroup_eventfd_list *ev, *tmp; 3555 3556 spin_lock(&memcg_oom_lock); 3557 3558 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) { 3559 if (ev->eventfd == eventfd) { 3560 list_del(&ev->list); 3561 kfree(ev); 3562 } 3563 } 3564 3565 spin_unlock(&memcg_oom_lock); 3566 } 3567 3568 static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v) 3569 { 3570 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf)); 3571 3572 seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable); 3573 seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom); 3574 return 0; 3575 } 3576 3577 static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css, 3578 struct cftype *cft, u64 val) 3579 { 3580 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 3581 3582 /* cannot set to root cgroup and only 0 and 1 are allowed */ 3583 if (!css->parent || !((val == 0) || (val == 1))) 3584 return -EINVAL; 3585 3586 memcg->oom_kill_disable = val; 3587 if (!val) 3588 memcg_oom_recover(memcg); 3589 3590 return 0; 3591 } 3592 3593 #ifdef CONFIG_CGROUP_WRITEBACK 3594 3595 struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg) 3596 { 3597 return &memcg->cgwb_list; 3598 } 3599 3600 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp) 3601 { 3602 return wb_domain_init(&memcg->cgwb_domain, gfp); 3603 } 3604 3605 static void memcg_wb_domain_exit(struct mem_cgroup *memcg) 3606 { 3607 wb_domain_exit(&memcg->cgwb_domain); 3608 } 3609 3610 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg) 3611 { 3612 wb_domain_size_changed(&memcg->cgwb_domain); 3613 } 3614 3615 struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb) 3616 { 3617 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); 3618 3619 if (!memcg->css.parent) 3620 return NULL; 3621 3622 return &memcg->cgwb_domain; 3623 } 3624 3625 /** 3626 * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg 3627 * @wb: bdi_writeback in question 3628 * @pfilepages: out parameter for number of file pages 3629 * @pheadroom: out parameter for number of allocatable pages according to memcg 3630 * @pdirty: out parameter for number of dirty pages 3631 * @pwriteback: out parameter for number of pages under writeback 3632 * 3633 * Determine the numbers of file, headroom, dirty, and writeback pages in 3634 * @wb's memcg. File, dirty and writeback are self-explanatory. Headroom 3635 * is a bit more involved. 3636 * 3637 * A memcg's headroom is "min(max, high) - used". In the hierarchy, the 3638 * headroom is calculated as the lowest headroom of itself and the 3639 * ancestors. Note that this doesn't consider the actual amount of 3640 * available memory in the system. The caller should further cap 3641 * *@pheadroom accordingly. 3642 */ 3643 void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages, 3644 unsigned long *pheadroom, unsigned long *pdirty, 3645 unsigned long *pwriteback) 3646 { 3647 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); 3648 struct mem_cgroup *parent; 3649 3650 *pdirty = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_DIRTY); 3651 3652 /* this should eventually include NR_UNSTABLE_NFS */ 3653 *pwriteback = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_WRITEBACK); 3654 *pfilepages = mem_cgroup_nr_lru_pages(memcg, (1 << LRU_INACTIVE_FILE) | 3655 (1 << LRU_ACTIVE_FILE)); 3656 *pheadroom = PAGE_COUNTER_MAX; 3657 3658 while ((parent = parent_mem_cgroup(memcg))) { 3659 unsigned long ceiling = min(memcg->memory.limit, memcg->high); 3660 unsigned long used = page_counter_read(&memcg->memory); 3661 3662 *pheadroom = min(*pheadroom, ceiling - min(ceiling, used)); 3663 memcg = parent; 3664 } 3665 } 3666 3667 #else /* CONFIG_CGROUP_WRITEBACK */ 3668 3669 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp) 3670 { 3671 return 0; 3672 } 3673 3674 static void memcg_wb_domain_exit(struct mem_cgroup *memcg) 3675 { 3676 } 3677 3678 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg) 3679 { 3680 } 3681 3682 #endif /* CONFIG_CGROUP_WRITEBACK */ 3683 3684 /* 3685 * DO NOT USE IN NEW FILES. 3686 * 3687 * "cgroup.event_control" implementation. 3688 * 3689 * This is way over-engineered. It tries to support fully configurable 3690 * events for each user. Such level of flexibility is completely 3691 * unnecessary especially in the light of the planned unified hierarchy. 3692 * 3693 * Please deprecate this and replace with something simpler if at all 3694 * possible. 3695 */ 3696 3697 /* 3698 * Unregister event and free resources. 3699 * 3700 * Gets called from workqueue. 3701 */ 3702 static void memcg_event_remove(struct work_struct *work) 3703 { 3704 struct mem_cgroup_event *event = 3705 container_of(work, struct mem_cgroup_event, remove); 3706 struct mem_cgroup *memcg = event->memcg; 3707 3708 remove_wait_queue(event->wqh, &event->wait); 3709 3710 event->unregister_event(memcg, event->eventfd); 3711 3712 /* Notify userspace the event is going away. */ 3713 eventfd_signal(event->eventfd, 1); 3714 3715 eventfd_ctx_put(event->eventfd); 3716 kfree(event); 3717 css_put(&memcg->css); 3718 } 3719 3720 /* 3721 * Gets called on POLLHUP on eventfd when user closes it. 3722 * 3723 * Called with wqh->lock held and interrupts disabled. 3724 */ 3725 static int memcg_event_wake(wait_queue_t *wait, unsigned mode, 3726 int sync, void *key) 3727 { 3728 struct mem_cgroup_event *event = 3729 container_of(wait, struct mem_cgroup_event, wait); 3730 struct mem_cgroup *memcg = event->memcg; 3731 unsigned long flags = (unsigned long)key; 3732 3733 if (flags & POLLHUP) { 3734 /* 3735 * If the event has been detached at cgroup removal, we 3736 * can simply return knowing the other side will cleanup 3737 * for us. 3738 * 3739 * We can't race against event freeing since the other 3740 * side will require wqh->lock via remove_wait_queue(), 3741 * which we hold. 3742 */ 3743 spin_lock(&memcg->event_list_lock); 3744 if (!list_empty(&event->list)) { 3745 list_del_init(&event->list); 3746 /* 3747 * We are in atomic context, but cgroup_event_remove() 3748 * may sleep, so we have to call it in workqueue. 3749 */ 3750 schedule_work(&event->remove); 3751 } 3752 spin_unlock(&memcg->event_list_lock); 3753 } 3754 3755 return 0; 3756 } 3757 3758 static void memcg_event_ptable_queue_proc(struct file *file, 3759 wait_queue_head_t *wqh, poll_table *pt) 3760 { 3761 struct mem_cgroup_event *event = 3762 container_of(pt, struct mem_cgroup_event, pt); 3763 3764 event->wqh = wqh; 3765 add_wait_queue(wqh, &event->wait); 3766 } 3767 3768 /* 3769 * DO NOT USE IN NEW FILES. 3770 * 3771 * Parse input and register new cgroup event handler. 3772 * 3773 * Input must be in format '<event_fd> <control_fd> <args>'. 3774 * Interpretation of args is defined by control file implementation. 3775 */ 3776 static ssize_t memcg_write_event_control(struct kernfs_open_file *of, 3777 char *buf, size_t nbytes, loff_t off) 3778 { 3779 struct cgroup_subsys_state *css = of_css(of); 3780 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 3781 struct mem_cgroup_event *event; 3782 struct cgroup_subsys_state *cfile_css; 3783 unsigned int efd, cfd; 3784 struct fd efile; 3785 struct fd cfile; 3786 const char *name; 3787 char *endp; 3788 int ret; 3789 3790 buf = strstrip(buf); 3791 3792 efd = simple_strtoul(buf, &endp, 10); 3793 if (*endp != ' ') 3794 return -EINVAL; 3795 buf = endp + 1; 3796 3797 cfd = simple_strtoul(buf, &endp, 10); 3798 if ((*endp != ' ') && (*endp != '\0')) 3799 return -EINVAL; 3800 buf = endp + 1; 3801 3802 event = kzalloc(sizeof(*event), GFP_KERNEL); 3803 if (!event) 3804 return -ENOMEM; 3805 3806 event->memcg = memcg; 3807 INIT_LIST_HEAD(&event->list); 3808 init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc); 3809 init_waitqueue_func_entry(&event->wait, memcg_event_wake); 3810 INIT_WORK(&event->remove, memcg_event_remove); 3811 3812 efile = fdget(efd); 3813 if (!efile.file) { 3814 ret = -EBADF; 3815 goto out_kfree; 3816 } 3817 3818 event->eventfd = eventfd_ctx_fileget(efile.file); 3819 if (IS_ERR(event->eventfd)) { 3820 ret = PTR_ERR(event->eventfd); 3821 goto out_put_efile; 3822 } 3823 3824 cfile = fdget(cfd); 3825 if (!cfile.file) { 3826 ret = -EBADF; 3827 goto out_put_eventfd; 3828 } 3829 3830 /* the process need read permission on control file */ 3831 /* AV: shouldn't we check that it's been opened for read instead? */ 3832 ret = inode_permission(file_inode(cfile.file), MAY_READ); 3833 if (ret < 0) 3834 goto out_put_cfile; 3835 3836 /* 3837 * Determine the event callbacks and set them in @event. This used 3838 * to be done via struct cftype but cgroup core no longer knows 3839 * about these events. The following is crude but the whole thing 3840 * is for compatibility anyway. 3841 * 3842 * DO NOT ADD NEW FILES. 3843 */ 3844 name = cfile.file->f_path.dentry->d_name.name; 3845 3846 if (!strcmp(name, "memory.usage_in_bytes")) { 3847 event->register_event = mem_cgroup_usage_register_event; 3848 event->unregister_event = mem_cgroup_usage_unregister_event; 3849 } else if (!strcmp(name, "memory.oom_control")) { 3850 event->register_event = mem_cgroup_oom_register_event; 3851 event->unregister_event = mem_cgroup_oom_unregister_event; 3852 } else if (!strcmp(name, "memory.pressure_level")) { 3853 event->register_event = vmpressure_register_event; 3854 event->unregister_event = vmpressure_unregister_event; 3855 } else if (!strcmp(name, "memory.memsw.usage_in_bytes")) { 3856 event->register_event = memsw_cgroup_usage_register_event; 3857 event->unregister_event = memsw_cgroup_usage_unregister_event; 3858 } else { 3859 ret = -EINVAL; 3860 goto out_put_cfile; 3861 } 3862 3863 /* 3864 * Verify @cfile should belong to @css. Also, remaining events are 3865 * automatically removed on cgroup destruction but the removal is 3866 * asynchronous, so take an extra ref on @css. 3867 */ 3868 cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent, 3869 &memory_cgrp_subsys); 3870 ret = -EINVAL; 3871 if (IS_ERR(cfile_css)) 3872 goto out_put_cfile; 3873 if (cfile_css != css) { 3874 css_put(cfile_css); 3875 goto out_put_cfile; 3876 } 3877 3878 ret = event->register_event(memcg, event->eventfd, buf); 3879 if (ret) 3880 goto out_put_css; 3881 3882 efile.file->f_op->poll(efile.file, &event->pt); 3883 3884 spin_lock(&memcg->event_list_lock); 3885 list_add(&event->list, &memcg->event_list); 3886 spin_unlock(&memcg->event_list_lock); 3887 3888 fdput(cfile); 3889 fdput(efile); 3890 3891 return nbytes; 3892 3893 out_put_css: 3894 css_put(css); 3895 out_put_cfile: 3896 fdput(cfile); 3897 out_put_eventfd: 3898 eventfd_ctx_put(event->eventfd); 3899 out_put_efile: 3900 fdput(efile); 3901 out_kfree: 3902 kfree(event); 3903 3904 return ret; 3905 } 3906 3907 static struct cftype mem_cgroup_legacy_files[] = { 3908 { 3909 .name = "usage_in_bytes", 3910 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE), 3911 .read_u64 = mem_cgroup_read_u64, 3912 }, 3913 { 3914 .name = "max_usage_in_bytes", 3915 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE), 3916 .write = mem_cgroup_reset, 3917 .read_u64 = mem_cgroup_read_u64, 3918 }, 3919 { 3920 .name = "limit_in_bytes", 3921 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT), 3922 .write = mem_cgroup_write, 3923 .read_u64 = mem_cgroup_read_u64, 3924 }, 3925 { 3926 .name = "soft_limit_in_bytes", 3927 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT), 3928 .write = mem_cgroup_write, 3929 .read_u64 = mem_cgroup_read_u64, 3930 }, 3931 { 3932 .name = "failcnt", 3933 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT), 3934 .write = mem_cgroup_reset, 3935 .read_u64 = mem_cgroup_read_u64, 3936 }, 3937 { 3938 .name = "stat", 3939 .seq_show = memcg_stat_show, 3940 }, 3941 { 3942 .name = "force_empty", 3943 .write = mem_cgroup_force_empty_write, 3944 }, 3945 { 3946 .name = "use_hierarchy", 3947 .write_u64 = mem_cgroup_hierarchy_write, 3948 .read_u64 = mem_cgroup_hierarchy_read, 3949 }, 3950 { 3951 .name = "cgroup.event_control", /* XXX: for compat */ 3952 .write = memcg_write_event_control, 3953 .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE, 3954 }, 3955 { 3956 .name = "swappiness", 3957 .read_u64 = mem_cgroup_swappiness_read, 3958 .write_u64 = mem_cgroup_swappiness_write, 3959 }, 3960 { 3961 .name = "move_charge_at_immigrate", 3962 .read_u64 = mem_cgroup_move_charge_read, 3963 .write_u64 = mem_cgroup_move_charge_write, 3964 }, 3965 { 3966 .name = "oom_control", 3967 .seq_show = mem_cgroup_oom_control_read, 3968 .write_u64 = mem_cgroup_oom_control_write, 3969 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL), 3970 }, 3971 { 3972 .name = "pressure_level", 3973 }, 3974 #ifdef CONFIG_NUMA 3975 { 3976 .name = "numa_stat", 3977 .seq_show = memcg_numa_stat_show, 3978 }, 3979 #endif 3980 { 3981 .name = "kmem.limit_in_bytes", 3982 .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT), 3983 .write = mem_cgroup_write, 3984 .read_u64 = mem_cgroup_read_u64, 3985 }, 3986 { 3987 .name = "kmem.usage_in_bytes", 3988 .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE), 3989 .read_u64 = mem_cgroup_read_u64, 3990 }, 3991 { 3992 .name = "kmem.failcnt", 3993 .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT), 3994 .write = mem_cgroup_reset, 3995 .read_u64 = mem_cgroup_read_u64, 3996 }, 3997 { 3998 .name = "kmem.max_usage_in_bytes", 3999 .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE), 4000 .write = mem_cgroup_reset, 4001 .read_u64 = mem_cgroup_read_u64, 4002 }, 4003 #ifdef CONFIG_SLABINFO 4004 { 4005 .name = "kmem.slabinfo", 4006 .seq_start = slab_start, 4007 .seq_next = slab_next, 4008 .seq_stop = slab_stop, 4009 .seq_show = memcg_slab_show, 4010 }, 4011 #endif 4012 { 4013 .name = "kmem.tcp.limit_in_bytes", 4014 .private = MEMFILE_PRIVATE(_TCP, RES_LIMIT), 4015 .write = mem_cgroup_write, 4016 .read_u64 = mem_cgroup_read_u64, 4017 }, 4018 { 4019 .name = "kmem.tcp.usage_in_bytes", 4020 .private = MEMFILE_PRIVATE(_TCP, RES_USAGE), 4021 .read_u64 = mem_cgroup_read_u64, 4022 }, 4023 { 4024 .name = "kmem.tcp.failcnt", 4025 .private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT), 4026 .write = mem_cgroup_reset, 4027 .read_u64 = mem_cgroup_read_u64, 4028 }, 4029 { 4030 .name = "kmem.tcp.max_usage_in_bytes", 4031 .private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE), 4032 .write = mem_cgroup_reset, 4033 .read_u64 = mem_cgroup_read_u64, 4034 }, 4035 { }, /* terminate */ 4036 }; 4037 4038 static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node) 4039 { 4040 struct mem_cgroup_per_node *pn; 4041 struct mem_cgroup_per_zone *mz; 4042 int zone, tmp = node; 4043 /* 4044 * This routine is called against possible nodes. 4045 * But it's BUG to call kmalloc() against offline node. 4046 * 4047 * TODO: this routine can waste much memory for nodes which will 4048 * never be onlined. It's better to use memory hotplug callback 4049 * function. 4050 */ 4051 if (!node_state(node, N_NORMAL_MEMORY)) 4052 tmp = -1; 4053 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp); 4054 if (!pn) 4055 return 1; 4056 4057 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 4058 mz = &pn->zoneinfo[zone]; 4059 lruvec_init(&mz->lruvec); 4060 mz->usage_in_excess = 0; 4061 mz->on_tree = false; 4062 mz->memcg = memcg; 4063 } 4064 memcg->nodeinfo[node] = pn; 4065 return 0; 4066 } 4067 4068 static void free_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node) 4069 { 4070 kfree(memcg->nodeinfo[node]); 4071 } 4072 4073 static void mem_cgroup_free(struct mem_cgroup *memcg) 4074 { 4075 int node; 4076 4077 memcg_wb_domain_exit(memcg); 4078 for_each_node(node) 4079 free_mem_cgroup_per_zone_info(memcg, node); 4080 free_percpu(memcg->stat); 4081 kfree(memcg); 4082 } 4083 4084 static struct mem_cgroup *mem_cgroup_alloc(void) 4085 { 4086 struct mem_cgroup *memcg; 4087 size_t size; 4088 int node; 4089 4090 size = sizeof(struct mem_cgroup); 4091 size += nr_node_ids * sizeof(struct mem_cgroup_per_node *); 4092 4093 memcg = kzalloc(size, GFP_KERNEL); 4094 if (!memcg) 4095 return NULL; 4096 4097 memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu); 4098 if (!memcg->stat) 4099 goto fail; 4100 4101 for_each_node(node) 4102 if (alloc_mem_cgroup_per_zone_info(memcg, node)) 4103 goto fail; 4104 4105 if (memcg_wb_domain_init(memcg, GFP_KERNEL)) 4106 goto fail; 4107 4108 INIT_WORK(&memcg->high_work, high_work_func); 4109 memcg->last_scanned_node = MAX_NUMNODES; 4110 INIT_LIST_HEAD(&memcg->oom_notify); 4111 mutex_init(&memcg->thresholds_lock); 4112 spin_lock_init(&memcg->move_lock); 4113 vmpressure_init(&memcg->vmpressure); 4114 INIT_LIST_HEAD(&memcg->event_list); 4115 spin_lock_init(&memcg->event_list_lock); 4116 memcg->socket_pressure = jiffies; 4117 #ifndef CONFIG_SLOB 4118 memcg->kmemcg_id = -1; 4119 #endif 4120 #ifdef CONFIG_CGROUP_WRITEBACK 4121 INIT_LIST_HEAD(&memcg->cgwb_list); 4122 #endif 4123 return memcg; 4124 fail: 4125 mem_cgroup_free(memcg); 4126 return NULL; 4127 } 4128 4129 static struct cgroup_subsys_state * __ref 4130 mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 4131 { 4132 struct mem_cgroup *parent = mem_cgroup_from_css(parent_css); 4133 struct mem_cgroup *memcg; 4134 long error = -ENOMEM; 4135 4136 memcg = mem_cgroup_alloc(); 4137 if (!memcg) 4138 return ERR_PTR(error); 4139 4140 memcg->high = PAGE_COUNTER_MAX; 4141 memcg->soft_limit = PAGE_COUNTER_MAX; 4142 if (parent) { 4143 memcg->swappiness = mem_cgroup_swappiness(parent); 4144 memcg->oom_kill_disable = parent->oom_kill_disable; 4145 } 4146 if (parent && parent->use_hierarchy) { 4147 memcg->use_hierarchy = true; 4148 page_counter_init(&memcg->memory, &parent->memory); 4149 page_counter_init(&memcg->swap, &parent->swap); 4150 page_counter_init(&memcg->memsw, &parent->memsw); 4151 page_counter_init(&memcg->kmem, &parent->kmem); 4152 page_counter_init(&memcg->tcpmem, &parent->tcpmem); 4153 } else { 4154 page_counter_init(&memcg->memory, NULL); 4155 page_counter_init(&memcg->swap, NULL); 4156 page_counter_init(&memcg->memsw, NULL); 4157 page_counter_init(&memcg->kmem, NULL); 4158 page_counter_init(&memcg->tcpmem, NULL); 4159 /* 4160 * Deeper hierachy with use_hierarchy == false doesn't make 4161 * much sense so let cgroup subsystem know about this 4162 * unfortunate state in our controller. 4163 */ 4164 if (parent != root_mem_cgroup) 4165 memory_cgrp_subsys.broken_hierarchy = true; 4166 } 4167 4168 /* The following stuff does not apply to the root */ 4169 if (!parent) { 4170 root_mem_cgroup = memcg; 4171 return &memcg->css; 4172 } 4173 4174 error = memcg_online_kmem(memcg); 4175 if (error) 4176 goto fail; 4177 4178 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket) 4179 static_branch_inc(&memcg_sockets_enabled_key); 4180 4181 return &memcg->css; 4182 fail: 4183 mem_cgroup_free(memcg); 4184 return NULL; 4185 } 4186 4187 static int 4188 mem_cgroup_css_online(struct cgroup_subsys_state *css) 4189 { 4190 if (css->id > MEM_CGROUP_ID_MAX) 4191 return -ENOSPC; 4192 4193 return 0; 4194 } 4195 4196 static void mem_cgroup_css_offline(struct cgroup_subsys_state *css) 4197 { 4198 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4199 struct mem_cgroup_event *event, *tmp; 4200 4201 /* 4202 * Unregister events and notify userspace. 4203 * Notify userspace about cgroup removing only after rmdir of cgroup 4204 * directory to avoid race between userspace and kernelspace. 4205 */ 4206 spin_lock(&memcg->event_list_lock); 4207 list_for_each_entry_safe(event, tmp, &memcg->event_list, list) { 4208 list_del_init(&event->list); 4209 schedule_work(&event->remove); 4210 } 4211 spin_unlock(&memcg->event_list_lock); 4212 4213 memcg_offline_kmem(memcg); 4214 wb_memcg_offline(memcg); 4215 } 4216 4217 static void mem_cgroup_css_released(struct cgroup_subsys_state *css) 4218 { 4219 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4220 4221 invalidate_reclaim_iterators(memcg); 4222 } 4223 4224 static void mem_cgroup_css_free(struct cgroup_subsys_state *css) 4225 { 4226 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4227 4228 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket) 4229 static_branch_dec(&memcg_sockets_enabled_key); 4230 4231 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active) 4232 static_branch_dec(&memcg_sockets_enabled_key); 4233 4234 vmpressure_cleanup(&memcg->vmpressure); 4235 cancel_work_sync(&memcg->high_work); 4236 mem_cgroup_remove_from_trees(memcg); 4237 memcg_free_kmem(memcg); 4238 mem_cgroup_free(memcg); 4239 } 4240 4241 /** 4242 * mem_cgroup_css_reset - reset the states of a mem_cgroup 4243 * @css: the target css 4244 * 4245 * Reset the states of the mem_cgroup associated with @css. This is 4246 * invoked when the userland requests disabling on the default hierarchy 4247 * but the memcg is pinned through dependency. The memcg should stop 4248 * applying policies and should revert to the vanilla state as it may be 4249 * made visible again. 4250 * 4251 * The current implementation only resets the essential configurations. 4252 * This needs to be expanded to cover all the visible parts. 4253 */ 4254 static void mem_cgroup_css_reset(struct cgroup_subsys_state *css) 4255 { 4256 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4257 4258 page_counter_limit(&memcg->memory, PAGE_COUNTER_MAX); 4259 page_counter_limit(&memcg->swap, PAGE_COUNTER_MAX); 4260 page_counter_limit(&memcg->memsw, PAGE_COUNTER_MAX); 4261 page_counter_limit(&memcg->kmem, PAGE_COUNTER_MAX); 4262 page_counter_limit(&memcg->tcpmem, PAGE_COUNTER_MAX); 4263 memcg->low = 0; 4264 memcg->high = PAGE_COUNTER_MAX; 4265 memcg->soft_limit = PAGE_COUNTER_MAX; 4266 memcg_wb_domain_size_changed(memcg); 4267 } 4268 4269 #ifdef CONFIG_MMU 4270 /* Handlers for move charge at task migration. */ 4271 static int mem_cgroup_do_precharge(unsigned long count) 4272 { 4273 int ret; 4274 4275 /* Try a single bulk charge without reclaim first, kswapd may wake */ 4276 ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count); 4277 if (!ret) { 4278 mc.precharge += count; 4279 return ret; 4280 } 4281 4282 /* Try charges one by one with reclaim */ 4283 while (count--) { 4284 ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_NORETRY, 1); 4285 if (ret) 4286 return ret; 4287 mc.precharge++; 4288 cond_resched(); 4289 } 4290 return 0; 4291 } 4292 4293 /** 4294 * get_mctgt_type - get target type of moving charge 4295 * @vma: the vma the pte to be checked belongs 4296 * @addr: the address corresponding to the pte to be checked 4297 * @ptent: the pte to be checked 4298 * @target: the pointer the target page or swap ent will be stored(can be NULL) 4299 * 4300 * Returns 4301 * 0(MC_TARGET_NONE): if the pte is not a target for move charge. 4302 * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for 4303 * move charge. if @target is not NULL, the page is stored in target->page 4304 * with extra refcnt got(Callers should handle it). 4305 * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a 4306 * target for charge migration. if @target is not NULL, the entry is stored 4307 * in target->ent. 4308 * 4309 * Called with pte lock held. 4310 */ 4311 union mc_target { 4312 struct page *page; 4313 swp_entry_t ent; 4314 }; 4315 4316 enum mc_target_type { 4317 MC_TARGET_NONE = 0, 4318 MC_TARGET_PAGE, 4319 MC_TARGET_SWAP, 4320 }; 4321 4322 static struct page *mc_handle_present_pte(struct vm_area_struct *vma, 4323 unsigned long addr, pte_t ptent) 4324 { 4325 struct page *page = vm_normal_page(vma, addr, ptent); 4326 4327 if (!page || !page_mapped(page)) 4328 return NULL; 4329 if (PageAnon(page)) { 4330 if (!(mc.flags & MOVE_ANON)) 4331 return NULL; 4332 } else { 4333 if (!(mc.flags & MOVE_FILE)) 4334 return NULL; 4335 } 4336 if (!get_page_unless_zero(page)) 4337 return NULL; 4338 4339 return page; 4340 } 4341 4342 #ifdef CONFIG_SWAP 4343 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma, 4344 unsigned long addr, pte_t ptent, swp_entry_t *entry) 4345 { 4346 struct page *page = NULL; 4347 swp_entry_t ent = pte_to_swp_entry(ptent); 4348 4349 if (!(mc.flags & MOVE_ANON) || non_swap_entry(ent)) 4350 return NULL; 4351 /* 4352 * Because lookup_swap_cache() updates some statistics counter, 4353 * we call find_get_page() with swapper_space directly. 4354 */ 4355 page = find_get_page(swap_address_space(ent), ent.val); 4356 if (do_memsw_account()) 4357 entry->val = ent.val; 4358 4359 return page; 4360 } 4361 #else 4362 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma, 4363 unsigned long addr, pte_t ptent, swp_entry_t *entry) 4364 { 4365 return NULL; 4366 } 4367 #endif 4368 4369 static struct page *mc_handle_file_pte(struct vm_area_struct *vma, 4370 unsigned long addr, pte_t ptent, swp_entry_t *entry) 4371 { 4372 struct page *page = NULL; 4373 struct address_space *mapping; 4374 pgoff_t pgoff; 4375 4376 if (!vma->vm_file) /* anonymous vma */ 4377 return NULL; 4378 if (!(mc.flags & MOVE_FILE)) 4379 return NULL; 4380 4381 mapping = vma->vm_file->f_mapping; 4382 pgoff = linear_page_index(vma, addr); 4383 4384 /* page is moved even if it's not RSS of this task(page-faulted). */ 4385 #ifdef CONFIG_SWAP 4386 /* shmem/tmpfs may report page out on swap: account for that too. */ 4387 if (shmem_mapping(mapping)) { 4388 page = find_get_entry(mapping, pgoff); 4389 if (radix_tree_exceptional_entry(page)) { 4390 swp_entry_t swp = radix_to_swp_entry(page); 4391 if (do_memsw_account()) 4392 *entry = swp; 4393 page = find_get_page(swap_address_space(swp), swp.val); 4394 } 4395 } else 4396 page = find_get_page(mapping, pgoff); 4397 #else 4398 page = find_get_page(mapping, pgoff); 4399 #endif 4400 return page; 4401 } 4402 4403 /** 4404 * mem_cgroup_move_account - move account of the page 4405 * @page: the page 4406 * @nr_pages: number of regular pages (>1 for huge pages) 4407 * @from: mem_cgroup which the page is moved from. 4408 * @to: mem_cgroup which the page is moved to. @from != @to. 4409 * 4410 * The caller must make sure the page is not on LRU (isolate_page() is useful.) 4411 * 4412 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge" 4413 * from old cgroup. 4414 */ 4415 static int mem_cgroup_move_account(struct page *page, 4416 bool compound, 4417 struct mem_cgroup *from, 4418 struct mem_cgroup *to) 4419 { 4420 unsigned long flags; 4421 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1; 4422 int ret; 4423 bool anon; 4424 4425 VM_BUG_ON(from == to); 4426 VM_BUG_ON_PAGE(PageLRU(page), page); 4427 VM_BUG_ON(compound && !PageTransHuge(page)); 4428 4429 /* 4430 * Prevent mem_cgroup_migrate() from looking at 4431 * page->mem_cgroup of its source page while we change it. 4432 */ 4433 ret = -EBUSY; 4434 if (!trylock_page(page)) 4435 goto out; 4436 4437 ret = -EINVAL; 4438 if (page->mem_cgroup != from) 4439 goto out_unlock; 4440 4441 anon = PageAnon(page); 4442 4443 spin_lock_irqsave(&from->move_lock, flags); 4444 4445 if (!anon && page_mapped(page)) { 4446 __this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED], 4447 nr_pages); 4448 __this_cpu_add(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED], 4449 nr_pages); 4450 } 4451 4452 /* 4453 * move_lock grabbed above and caller set from->moving_account, so 4454 * mem_cgroup_update_page_stat() will serialize updates to PageDirty. 4455 * So mapping should be stable for dirty pages. 4456 */ 4457 if (!anon && PageDirty(page)) { 4458 struct address_space *mapping = page_mapping(page); 4459 4460 if (mapping_cap_account_dirty(mapping)) { 4461 __this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_DIRTY], 4462 nr_pages); 4463 __this_cpu_add(to->stat->count[MEM_CGROUP_STAT_DIRTY], 4464 nr_pages); 4465 } 4466 } 4467 4468 if (PageWriteback(page)) { 4469 __this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_WRITEBACK], 4470 nr_pages); 4471 __this_cpu_add(to->stat->count[MEM_CGROUP_STAT_WRITEBACK], 4472 nr_pages); 4473 } 4474 4475 /* 4476 * It is safe to change page->mem_cgroup here because the page 4477 * is referenced, charged, and isolated - we can't race with 4478 * uncharging, charging, migration, or LRU putback. 4479 */ 4480 4481 /* caller should have done css_get */ 4482 page->mem_cgroup = to; 4483 spin_unlock_irqrestore(&from->move_lock, flags); 4484 4485 ret = 0; 4486 4487 local_irq_disable(); 4488 mem_cgroup_charge_statistics(to, page, compound, nr_pages); 4489 memcg_check_events(to, page); 4490 mem_cgroup_charge_statistics(from, page, compound, -nr_pages); 4491 memcg_check_events(from, page); 4492 local_irq_enable(); 4493 out_unlock: 4494 unlock_page(page); 4495 out: 4496 return ret; 4497 } 4498 4499 static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma, 4500 unsigned long addr, pte_t ptent, union mc_target *target) 4501 { 4502 struct page *page = NULL; 4503 enum mc_target_type ret = MC_TARGET_NONE; 4504 swp_entry_t ent = { .val = 0 }; 4505 4506 if (pte_present(ptent)) 4507 page = mc_handle_present_pte(vma, addr, ptent); 4508 else if (is_swap_pte(ptent)) 4509 page = mc_handle_swap_pte(vma, addr, ptent, &ent); 4510 else if (pte_none(ptent)) 4511 page = mc_handle_file_pte(vma, addr, ptent, &ent); 4512 4513 if (!page && !ent.val) 4514 return ret; 4515 if (page) { 4516 /* 4517 * Do only loose check w/o serialization. 4518 * mem_cgroup_move_account() checks the page is valid or 4519 * not under LRU exclusion. 4520 */ 4521 if (page->mem_cgroup == mc.from) { 4522 ret = MC_TARGET_PAGE; 4523 if (target) 4524 target->page = page; 4525 } 4526 if (!ret || !target) 4527 put_page(page); 4528 } 4529 /* There is a swap entry and a page doesn't exist or isn't charged */ 4530 if (ent.val && !ret && 4531 mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) { 4532 ret = MC_TARGET_SWAP; 4533 if (target) 4534 target->ent = ent; 4535 } 4536 return ret; 4537 } 4538 4539 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4540 /* 4541 * We don't consider swapping or file mapped pages because THP does not 4542 * support them for now. 4543 * Caller should make sure that pmd_trans_huge(pmd) is true. 4544 */ 4545 static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma, 4546 unsigned long addr, pmd_t pmd, union mc_target *target) 4547 { 4548 struct page *page = NULL; 4549 enum mc_target_type ret = MC_TARGET_NONE; 4550 4551 page = pmd_page(pmd); 4552 VM_BUG_ON_PAGE(!page || !PageHead(page), page); 4553 if (!(mc.flags & MOVE_ANON)) 4554 return ret; 4555 if (page->mem_cgroup == mc.from) { 4556 ret = MC_TARGET_PAGE; 4557 if (target) { 4558 get_page(page); 4559 target->page = page; 4560 } 4561 } 4562 return ret; 4563 } 4564 #else 4565 static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma, 4566 unsigned long addr, pmd_t pmd, union mc_target *target) 4567 { 4568 return MC_TARGET_NONE; 4569 } 4570 #endif 4571 4572 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd, 4573 unsigned long addr, unsigned long end, 4574 struct mm_walk *walk) 4575 { 4576 struct vm_area_struct *vma = walk->vma; 4577 pte_t *pte; 4578 spinlock_t *ptl; 4579 4580 ptl = pmd_trans_huge_lock(pmd, vma); 4581 if (ptl) { 4582 if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE) 4583 mc.precharge += HPAGE_PMD_NR; 4584 spin_unlock(ptl); 4585 return 0; 4586 } 4587 4588 if (pmd_trans_unstable(pmd)) 4589 return 0; 4590 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 4591 for (; addr != end; pte++, addr += PAGE_SIZE) 4592 if (get_mctgt_type(vma, addr, *pte, NULL)) 4593 mc.precharge++; /* increment precharge temporarily */ 4594 pte_unmap_unlock(pte - 1, ptl); 4595 cond_resched(); 4596 4597 return 0; 4598 } 4599 4600 static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm) 4601 { 4602 unsigned long precharge; 4603 4604 struct mm_walk mem_cgroup_count_precharge_walk = { 4605 .pmd_entry = mem_cgroup_count_precharge_pte_range, 4606 .mm = mm, 4607 }; 4608 down_read(&mm->mmap_sem); 4609 walk_page_range(0, ~0UL, &mem_cgroup_count_precharge_walk); 4610 up_read(&mm->mmap_sem); 4611 4612 precharge = mc.precharge; 4613 mc.precharge = 0; 4614 4615 return precharge; 4616 } 4617 4618 static int mem_cgroup_precharge_mc(struct mm_struct *mm) 4619 { 4620 unsigned long precharge = mem_cgroup_count_precharge(mm); 4621 4622 VM_BUG_ON(mc.moving_task); 4623 mc.moving_task = current; 4624 return mem_cgroup_do_precharge(precharge); 4625 } 4626 4627 /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */ 4628 static void __mem_cgroup_clear_mc(void) 4629 { 4630 struct mem_cgroup *from = mc.from; 4631 struct mem_cgroup *to = mc.to; 4632 4633 /* we must uncharge all the leftover precharges from mc.to */ 4634 if (mc.precharge) { 4635 cancel_charge(mc.to, mc.precharge); 4636 mc.precharge = 0; 4637 } 4638 /* 4639 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so 4640 * we must uncharge here. 4641 */ 4642 if (mc.moved_charge) { 4643 cancel_charge(mc.from, mc.moved_charge); 4644 mc.moved_charge = 0; 4645 } 4646 /* we must fixup refcnts and charges */ 4647 if (mc.moved_swap) { 4648 /* uncharge swap account from the old cgroup */ 4649 if (!mem_cgroup_is_root(mc.from)) 4650 page_counter_uncharge(&mc.from->memsw, mc.moved_swap); 4651 4652 /* 4653 * we charged both to->memory and to->memsw, so we 4654 * should uncharge to->memory. 4655 */ 4656 if (!mem_cgroup_is_root(mc.to)) 4657 page_counter_uncharge(&mc.to->memory, mc.moved_swap); 4658 4659 css_put_many(&mc.from->css, mc.moved_swap); 4660 4661 /* we've already done css_get(mc.to) */ 4662 mc.moved_swap = 0; 4663 } 4664 memcg_oom_recover(from); 4665 memcg_oom_recover(to); 4666 wake_up_all(&mc.waitq); 4667 } 4668 4669 static void mem_cgroup_clear_mc(void) 4670 { 4671 struct mm_struct *mm = mc.mm; 4672 4673 /* 4674 * we must clear moving_task before waking up waiters at the end of 4675 * task migration. 4676 */ 4677 mc.moving_task = NULL; 4678 __mem_cgroup_clear_mc(); 4679 spin_lock(&mc.lock); 4680 mc.from = NULL; 4681 mc.to = NULL; 4682 mc.mm = NULL; 4683 spin_unlock(&mc.lock); 4684 4685 mmput(mm); 4686 } 4687 4688 static int mem_cgroup_can_attach(struct cgroup_taskset *tset) 4689 { 4690 struct cgroup_subsys_state *css; 4691 struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */ 4692 struct mem_cgroup *from; 4693 struct task_struct *leader, *p; 4694 struct mm_struct *mm; 4695 unsigned long move_flags; 4696 int ret = 0; 4697 4698 /* charge immigration isn't supported on the default hierarchy */ 4699 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 4700 return 0; 4701 4702 /* 4703 * Multi-process migrations only happen on the default hierarchy 4704 * where charge immigration is not used. Perform charge 4705 * immigration if @tset contains a leader and whine if there are 4706 * multiple. 4707 */ 4708 p = NULL; 4709 cgroup_taskset_for_each_leader(leader, css, tset) { 4710 WARN_ON_ONCE(p); 4711 p = leader; 4712 memcg = mem_cgroup_from_css(css); 4713 } 4714 if (!p) 4715 return 0; 4716 4717 /* 4718 * We are now commited to this value whatever it is. Changes in this 4719 * tunable will only affect upcoming migrations, not the current one. 4720 * So we need to save it, and keep it going. 4721 */ 4722 move_flags = READ_ONCE(memcg->move_charge_at_immigrate); 4723 if (!move_flags) 4724 return 0; 4725 4726 from = mem_cgroup_from_task(p); 4727 4728 VM_BUG_ON(from == memcg); 4729 4730 mm = get_task_mm(p); 4731 if (!mm) 4732 return 0; 4733 /* We move charges only when we move a owner of the mm */ 4734 if (mm->owner == p) { 4735 VM_BUG_ON(mc.from); 4736 VM_BUG_ON(mc.to); 4737 VM_BUG_ON(mc.precharge); 4738 VM_BUG_ON(mc.moved_charge); 4739 VM_BUG_ON(mc.moved_swap); 4740 4741 spin_lock(&mc.lock); 4742 mc.mm = mm; 4743 mc.from = from; 4744 mc.to = memcg; 4745 mc.flags = move_flags; 4746 spin_unlock(&mc.lock); 4747 /* We set mc.moving_task later */ 4748 4749 ret = mem_cgroup_precharge_mc(mm); 4750 if (ret) 4751 mem_cgroup_clear_mc(); 4752 } else { 4753 mmput(mm); 4754 } 4755 return ret; 4756 } 4757 4758 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset) 4759 { 4760 if (mc.to) 4761 mem_cgroup_clear_mc(); 4762 } 4763 4764 static int mem_cgroup_move_charge_pte_range(pmd_t *pmd, 4765 unsigned long addr, unsigned long end, 4766 struct mm_walk *walk) 4767 { 4768 int ret = 0; 4769 struct vm_area_struct *vma = walk->vma; 4770 pte_t *pte; 4771 spinlock_t *ptl; 4772 enum mc_target_type target_type; 4773 union mc_target target; 4774 struct page *page; 4775 4776 ptl = pmd_trans_huge_lock(pmd, vma); 4777 if (ptl) { 4778 if (mc.precharge < HPAGE_PMD_NR) { 4779 spin_unlock(ptl); 4780 return 0; 4781 } 4782 target_type = get_mctgt_type_thp(vma, addr, *pmd, &target); 4783 if (target_type == MC_TARGET_PAGE) { 4784 page = target.page; 4785 if (!isolate_lru_page(page)) { 4786 if (!mem_cgroup_move_account(page, true, 4787 mc.from, mc.to)) { 4788 mc.precharge -= HPAGE_PMD_NR; 4789 mc.moved_charge += HPAGE_PMD_NR; 4790 } 4791 putback_lru_page(page); 4792 } 4793 put_page(page); 4794 } 4795 spin_unlock(ptl); 4796 return 0; 4797 } 4798 4799 if (pmd_trans_unstable(pmd)) 4800 return 0; 4801 retry: 4802 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 4803 for (; addr != end; addr += PAGE_SIZE) { 4804 pte_t ptent = *(pte++); 4805 swp_entry_t ent; 4806 4807 if (!mc.precharge) 4808 break; 4809 4810 switch (get_mctgt_type(vma, addr, ptent, &target)) { 4811 case MC_TARGET_PAGE: 4812 page = target.page; 4813 /* 4814 * We can have a part of the split pmd here. Moving it 4815 * can be done but it would be too convoluted so simply 4816 * ignore such a partial THP and keep it in original 4817 * memcg. There should be somebody mapping the head. 4818 */ 4819 if (PageTransCompound(page)) 4820 goto put; 4821 if (isolate_lru_page(page)) 4822 goto put; 4823 if (!mem_cgroup_move_account(page, false, 4824 mc.from, mc.to)) { 4825 mc.precharge--; 4826 /* we uncharge from mc.from later. */ 4827 mc.moved_charge++; 4828 } 4829 putback_lru_page(page); 4830 put: /* get_mctgt_type() gets the page */ 4831 put_page(page); 4832 break; 4833 case MC_TARGET_SWAP: 4834 ent = target.ent; 4835 if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) { 4836 mc.precharge--; 4837 /* we fixup refcnts and charges later. */ 4838 mc.moved_swap++; 4839 } 4840 break; 4841 default: 4842 break; 4843 } 4844 } 4845 pte_unmap_unlock(pte - 1, ptl); 4846 cond_resched(); 4847 4848 if (addr != end) { 4849 /* 4850 * We have consumed all precharges we got in can_attach(). 4851 * We try charge one by one, but don't do any additional 4852 * charges to mc.to if we have failed in charge once in attach() 4853 * phase. 4854 */ 4855 ret = mem_cgroup_do_precharge(1); 4856 if (!ret) 4857 goto retry; 4858 } 4859 4860 return ret; 4861 } 4862 4863 static void mem_cgroup_move_charge(void) 4864 { 4865 struct mm_walk mem_cgroup_move_charge_walk = { 4866 .pmd_entry = mem_cgroup_move_charge_pte_range, 4867 .mm = mc.mm, 4868 }; 4869 4870 lru_add_drain_all(); 4871 /* 4872 * Signal lock_page_memcg() to take the memcg's move_lock 4873 * while we're moving its pages to another memcg. Then wait 4874 * for already started RCU-only updates to finish. 4875 */ 4876 atomic_inc(&mc.from->moving_account); 4877 synchronize_rcu(); 4878 retry: 4879 if (unlikely(!down_read_trylock(&mc.mm->mmap_sem))) { 4880 /* 4881 * Someone who are holding the mmap_sem might be waiting in 4882 * waitq. So we cancel all extra charges, wake up all waiters, 4883 * and retry. Because we cancel precharges, we might not be able 4884 * to move enough charges, but moving charge is a best-effort 4885 * feature anyway, so it wouldn't be a big problem. 4886 */ 4887 __mem_cgroup_clear_mc(); 4888 cond_resched(); 4889 goto retry; 4890 } 4891 /* 4892 * When we have consumed all precharges and failed in doing 4893 * additional charge, the page walk just aborts. 4894 */ 4895 walk_page_range(0, ~0UL, &mem_cgroup_move_charge_walk); 4896 up_read(&mc.mm->mmap_sem); 4897 atomic_dec(&mc.from->moving_account); 4898 } 4899 4900 static void mem_cgroup_move_task(void) 4901 { 4902 if (mc.to) { 4903 mem_cgroup_move_charge(); 4904 mem_cgroup_clear_mc(); 4905 } 4906 } 4907 #else /* !CONFIG_MMU */ 4908 static int mem_cgroup_can_attach(struct cgroup_taskset *tset) 4909 { 4910 return 0; 4911 } 4912 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset) 4913 { 4914 } 4915 static void mem_cgroup_move_task(void) 4916 { 4917 } 4918 #endif 4919 4920 /* 4921 * Cgroup retains root cgroups across [un]mount cycles making it necessary 4922 * to verify whether we're attached to the default hierarchy on each mount 4923 * attempt. 4924 */ 4925 static void mem_cgroup_bind(struct cgroup_subsys_state *root_css) 4926 { 4927 /* 4928 * use_hierarchy is forced on the default hierarchy. cgroup core 4929 * guarantees that @root doesn't have any children, so turning it 4930 * on for the root memcg is enough. 4931 */ 4932 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 4933 root_mem_cgroup->use_hierarchy = true; 4934 else 4935 root_mem_cgroup->use_hierarchy = false; 4936 } 4937 4938 static u64 memory_current_read(struct cgroup_subsys_state *css, 4939 struct cftype *cft) 4940 { 4941 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4942 4943 return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE; 4944 } 4945 4946 static int memory_low_show(struct seq_file *m, void *v) 4947 { 4948 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 4949 unsigned long low = READ_ONCE(memcg->low); 4950 4951 if (low == PAGE_COUNTER_MAX) 4952 seq_puts(m, "max\n"); 4953 else 4954 seq_printf(m, "%llu\n", (u64)low * PAGE_SIZE); 4955 4956 return 0; 4957 } 4958 4959 static ssize_t memory_low_write(struct kernfs_open_file *of, 4960 char *buf, size_t nbytes, loff_t off) 4961 { 4962 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 4963 unsigned long low; 4964 int err; 4965 4966 buf = strstrip(buf); 4967 err = page_counter_memparse(buf, "max", &low); 4968 if (err) 4969 return err; 4970 4971 memcg->low = low; 4972 4973 return nbytes; 4974 } 4975 4976 static int memory_high_show(struct seq_file *m, void *v) 4977 { 4978 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 4979 unsigned long high = READ_ONCE(memcg->high); 4980 4981 if (high == PAGE_COUNTER_MAX) 4982 seq_puts(m, "max\n"); 4983 else 4984 seq_printf(m, "%llu\n", (u64)high * PAGE_SIZE); 4985 4986 return 0; 4987 } 4988 4989 static ssize_t memory_high_write(struct kernfs_open_file *of, 4990 char *buf, size_t nbytes, loff_t off) 4991 { 4992 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 4993 unsigned long nr_pages; 4994 unsigned long high; 4995 int err; 4996 4997 buf = strstrip(buf); 4998 err = page_counter_memparse(buf, "max", &high); 4999 if (err) 5000 return err; 5001 5002 memcg->high = high; 5003 5004 nr_pages = page_counter_read(&memcg->memory); 5005 if (nr_pages > high) 5006 try_to_free_mem_cgroup_pages(memcg, nr_pages - high, 5007 GFP_KERNEL, true); 5008 5009 memcg_wb_domain_size_changed(memcg); 5010 return nbytes; 5011 } 5012 5013 static int memory_max_show(struct seq_file *m, void *v) 5014 { 5015 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 5016 unsigned long max = READ_ONCE(memcg->memory.limit); 5017 5018 if (max == PAGE_COUNTER_MAX) 5019 seq_puts(m, "max\n"); 5020 else 5021 seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE); 5022 5023 return 0; 5024 } 5025 5026 static ssize_t memory_max_write(struct kernfs_open_file *of, 5027 char *buf, size_t nbytes, loff_t off) 5028 { 5029 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 5030 unsigned int nr_reclaims = MEM_CGROUP_RECLAIM_RETRIES; 5031 bool drained = false; 5032 unsigned long max; 5033 int err; 5034 5035 buf = strstrip(buf); 5036 err = page_counter_memparse(buf, "max", &max); 5037 if (err) 5038 return err; 5039 5040 xchg(&memcg->memory.limit, max); 5041 5042 for (;;) { 5043 unsigned long nr_pages = page_counter_read(&memcg->memory); 5044 5045 if (nr_pages <= max) 5046 break; 5047 5048 if (signal_pending(current)) { 5049 err = -EINTR; 5050 break; 5051 } 5052 5053 if (!drained) { 5054 drain_all_stock(memcg); 5055 drained = true; 5056 continue; 5057 } 5058 5059 if (nr_reclaims) { 5060 if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max, 5061 GFP_KERNEL, true)) 5062 nr_reclaims--; 5063 continue; 5064 } 5065 5066 mem_cgroup_events(memcg, MEMCG_OOM, 1); 5067 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0)) 5068 break; 5069 } 5070 5071 memcg_wb_domain_size_changed(memcg); 5072 return nbytes; 5073 } 5074 5075 static int memory_events_show(struct seq_file *m, void *v) 5076 { 5077 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 5078 5079 seq_printf(m, "low %lu\n", mem_cgroup_read_events(memcg, MEMCG_LOW)); 5080 seq_printf(m, "high %lu\n", mem_cgroup_read_events(memcg, MEMCG_HIGH)); 5081 seq_printf(m, "max %lu\n", mem_cgroup_read_events(memcg, MEMCG_MAX)); 5082 seq_printf(m, "oom %lu\n", mem_cgroup_read_events(memcg, MEMCG_OOM)); 5083 5084 return 0; 5085 } 5086 5087 static int memory_stat_show(struct seq_file *m, void *v) 5088 { 5089 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 5090 unsigned long stat[MEMCG_NR_STAT]; 5091 unsigned long events[MEMCG_NR_EVENTS]; 5092 int i; 5093 5094 /* 5095 * Provide statistics on the state of the memory subsystem as 5096 * well as cumulative event counters that show past behavior. 5097 * 5098 * This list is ordered following a combination of these gradients: 5099 * 1) generic big picture -> specifics and details 5100 * 2) reflecting userspace activity -> reflecting kernel heuristics 5101 * 5102 * Current memory state: 5103 */ 5104 5105 tree_stat(memcg, stat); 5106 tree_events(memcg, events); 5107 5108 seq_printf(m, "anon %llu\n", 5109 (u64)stat[MEM_CGROUP_STAT_RSS] * PAGE_SIZE); 5110 seq_printf(m, "file %llu\n", 5111 (u64)stat[MEM_CGROUP_STAT_CACHE] * PAGE_SIZE); 5112 seq_printf(m, "kernel_stack %llu\n", 5113 (u64)stat[MEMCG_KERNEL_STACK] * PAGE_SIZE); 5114 seq_printf(m, "slab %llu\n", 5115 (u64)(stat[MEMCG_SLAB_RECLAIMABLE] + 5116 stat[MEMCG_SLAB_UNRECLAIMABLE]) * PAGE_SIZE); 5117 seq_printf(m, "sock %llu\n", 5118 (u64)stat[MEMCG_SOCK] * PAGE_SIZE); 5119 5120 seq_printf(m, "file_mapped %llu\n", 5121 (u64)stat[MEM_CGROUP_STAT_FILE_MAPPED] * PAGE_SIZE); 5122 seq_printf(m, "file_dirty %llu\n", 5123 (u64)stat[MEM_CGROUP_STAT_DIRTY] * PAGE_SIZE); 5124 seq_printf(m, "file_writeback %llu\n", 5125 (u64)stat[MEM_CGROUP_STAT_WRITEBACK] * PAGE_SIZE); 5126 5127 for (i = 0; i < NR_LRU_LISTS; i++) { 5128 struct mem_cgroup *mi; 5129 unsigned long val = 0; 5130 5131 for_each_mem_cgroup_tree(mi, memcg) 5132 val += mem_cgroup_nr_lru_pages(mi, BIT(i)); 5133 seq_printf(m, "%s %llu\n", 5134 mem_cgroup_lru_names[i], (u64)val * PAGE_SIZE); 5135 } 5136 5137 seq_printf(m, "slab_reclaimable %llu\n", 5138 (u64)stat[MEMCG_SLAB_RECLAIMABLE] * PAGE_SIZE); 5139 seq_printf(m, "slab_unreclaimable %llu\n", 5140 (u64)stat[MEMCG_SLAB_UNRECLAIMABLE] * PAGE_SIZE); 5141 5142 /* Accumulated memory events */ 5143 5144 seq_printf(m, "pgfault %lu\n", 5145 events[MEM_CGROUP_EVENTS_PGFAULT]); 5146 seq_printf(m, "pgmajfault %lu\n", 5147 events[MEM_CGROUP_EVENTS_PGMAJFAULT]); 5148 5149 return 0; 5150 } 5151 5152 static struct cftype memory_files[] = { 5153 { 5154 .name = "current", 5155 .flags = CFTYPE_NOT_ON_ROOT, 5156 .read_u64 = memory_current_read, 5157 }, 5158 { 5159 .name = "low", 5160 .flags = CFTYPE_NOT_ON_ROOT, 5161 .seq_show = memory_low_show, 5162 .write = memory_low_write, 5163 }, 5164 { 5165 .name = "high", 5166 .flags = CFTYPE_NOT_ON_ROOT, 5167 .seq_show = memory_high_show, 5168 .write = memory_high_write, 5169 }, 5170 { 5171 .name = "max", 5172 .flags = CFTYPE_NOT_ON_ROOT, 5173 .seq_show = memory_max_show, 5174 .write = memory_max_write, 5175 }, 5176 { 5177 .name = "events", 5178 .flags = CFTYPE_NOT_ON_ROOT, 5179 .file_offset = offsetof(struct mem_cgroup, events_file), 5180 .seq_show = memory_events_show, 5181 }, 5182 { 5183 .name = "stat", 5184 .flags = CFTYPE_NOT_ON_ROOT, 5185 .seq_show = memory_stat_show, 5186 }, 5187 { } /* terminate */ 5188 }; 5189 5190 struct cgroup_subsys memory_cgrp_subsys = { 5191 .css_alloc = mem_cgroup_css_alloc, 5192 .css_online = mem_cgroup_css_online, 5193 .css_offline = mem_cgroup_css_offline, 5194 .css_released = mem_cgroup_css_released, 5195 .css_free = mem_cgroup_css_free, 5196 .css_reset = mem_cgroup_css_reset, 5197 .can_attach = mem_cgroup_can_attach, 5198 .cancel_attach = mem_cgroup_cancel_attach, 5199 .post_attach = mem_cgroup_move_task, 5200 .bind = mem_cgroup_bind, 5201 .dfl_cftypes = memory_files, 5202 .legacy_cftypes = mem_cgroup_legacy_files, 5203 .early_init = 0, 5204 }; 5205 5206 /** 5207 * mem_cgroup_low - check if memory consumption is below the normal range 5208 * @root: the highest ancestor to consider 5209 * @memcg: the memory cgroup to check 5210 * 5211 * Returns %true if memory consumption of @memcg, and that of all 5212 * configurable ancestors up to @root, is below the normal range. 5213 */ 5214 bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg) 5215 { 5216 if (mem_cgroup_disabled()) 5217 return false; 5218 5219 /* 5220 * The toplevel group doesn't have a configurable range, so 5221 * it's never low when looked at directly, and it is not 5222 * considered an ancestor when assessing the hierarchy. 5223 */ 5224 5225 if (memcg == root_mem_cgroup) 5226 return false; 5227 5228 if (page_counter_read(&memcg->memory) >= memcg->low) 5229 return false; 5230 5231 while (memcg != root) { 5232 memcg = parent_mem_cgroup(memcg); 5233 5234 if (memcg == root_mem_cgroup) 5235 break; 5236 5237 if (page_counter_read(&memcg->memory) >= memcg->low) 5238 return false; 5239 } 5240 return true; 5241 } 5242 5243 /** 5244 * mem_cgroup_try_charge - try charging a page 5245 * @page: page to charge 5246 * @mm: mm context of the victim 5247 * @gfp_mask: reclaim mode 5248 * @memcgp: charged memcg return 5249 * 5250 * Try to charge @page to the memcg that @mm belongs to, reclaiming 5251 * pages according to @gfp_mask if necessary. 5252 * 5253 * Returns 0 on success, with *@memcgp pointing to the charged memcg. 5254 * Otherwise, an error code is returned. 5255 * 5256 * After page->mapping has been set up, the caller must finalize the 5257 * charge with mem_cgroup_commit_charge(). Or abort the transaction 5258 * with mem_cgroup_cancel_charge() in case page instantiation fails. 5259 */ 5260 int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm, 5261 gfp_t gfp_mask, struct mem_cgroup **memcgp, 5262 bool compound) 5263 { 5264 struct mem_cgroup *memcg = NULL; 5265 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1; 5266 int ret = 0; 5267 5268 if (mem_cgroup_disabled()) 5269 goto out; 5270 5271 if (PageSwapCache(page)) { 5272 /* 5273 * Every swap fault against a single page tries to charge the 5274 * page, bail as early as possible. shmem_unuse() encounters 5275 * already charged pages, too. The USED bit is protected by 5276 * the page lock, which serializes swap cache removal, which 5277 * in turn serializes uncharging. 5278 */ 5279 VM_BUG_ON_PAGE(!PageLocked(page), page); 5280 if (page->mem_cgroup) 5281 goto out; 5282 5283 if (do_swap_account) { 5284 swp_entry_t ent = { .val = page_private(page), }; 5285 unsigned short id = lookup_swap_cgroup_id(ent); 5286 5287 rcu_read_lock(); 5288 memcg = mem_cgroup_from_id(id); 5289 if (memcg && !css_tryget_online(&memcg->css)) 5290 memcg = NULL; 5291 rcu_read_unlock(); 5292 } 5293 } 5294 5295 if (!memcg) 5296 memcg = get_mem_cgroup_from_mm(mm); 5297 5298 ret = try_charge(memcg, gfp_mask, nr_pages); 5299 5300 css_put(&memcg->css); 5301 out: 5302 *memcgp = memcg; 5303 return ret; 5304 } 5305 5306 /** 5307 * mem_cgroup_commit_charge - commit a page charge 5308 * @page: page to charge 5309 * @memcg: memcg to charge the page to 5310 * @lrucare: page might be on LRU already 5311 * 5312 * Finalize a charge transaction started by mem_cgroup_try_charge(), 5313 * after page->mapping has been set up. This must happen atomically 5314 * as part of the page instantiation, i.e. under the page table lock 5315 * for anonymous pages, under the page lock for page and swap cache. 5316 * 5317 * In addition, the page must not be on the LRU during the commit, to 5318 * prevent racing with task migration. If it might be, use @lrucare. 5319 * 5320 * Use mem_cgroup_cancel_charge() to cancel the transaction instead. 5321 */ 5322 void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg, 5323 bool lrucare, bool compound) 5324 { 5325 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1; 5326 5327 VM_BUG_ON_PAGE(!page->mapping, page); 5328 VM_BUG_ON_PAGE(PageLRU(page) && !lrucare, page); 5329 5330 if (mem_cgroup_disabled()) 5331 return; 5332 /* 5333 * Swap faults will attempt to charge the same page multiple 5334 * times. But reuse_swap_page() might have removed the page 5335 * from swapcache already, so we can't check PageSwapCache(). 5336 */ 5337 if (!memcg) 5338 return; 5339 5340 commit_charge(page, memcg, lrucare); 5341 5342 local_irq_disable(); 5343 mem_cgroup_charge_statistics(memcg, page, compound, nr_pages); 5344 memcg_check_events(memcg, page); 5345 local_irq_enable(); 5346 5347 if (do_memsw_account() && PageSwapCache(page)) { 5348 swp_entry_t entry = { .val = page_private(page) }; 5349 /* 5350 * The swap entry might not get freed for a long time, 5351 * let's not wait for it. The page already received a 5352 * memory+swap charge, drop the swap entry duplicate. 5353 */ 5354 mem_cgroup_uncharge_swap(entry); 5355 } 5356 } 5357 5358 /** 5359 * mem_cgroup_cancel_charge - cancel a page charge 5360 * @page: page to charge 5361 * @memcg: memcg to charge the page to 5362 * 5363 * Cancel a charge transaction started by mem_cgroup_try_charge(). 5364 */ 5365 void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg, 5366 bool compound) 5367 { 5368 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1; 5369 5370 if (mem_cgroup_disabled()) 5371 return; 5372 /* 5373 * Swap faults will attempt to charge the same page multiple 5374 * times. But reuse_swap_page() might have removed the page 5375 * from swapcache already, so we can't check PageSwapCache(). 5376 */ 5377 if (!memcg) 5378 return; 5379 5380 cancel_charge(memcg, nr_pages); 5381 } 5382 5383 static void uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout, 5384 unsigned long nr_anon, unsigned long nr_file, 5385 unsigned long nr_huge, struct page *dummy_page) 5386 { 5387 unsigned long nr_pages = nr_anon + nr_file; 5388 unsigned long flags; 5389 5390 if (!mem_cgroup_is_root(memcg)) { 5391 page_counter_uncharge(&memcg->memory, nr_pages); 5392 if (do_memsw_account()) 5393 page_counter_uncharge(&memcg->memsw, nr_pages); 5394 memcg_oom_recover(memcg); 5395 } 5396 5397 local_irq_save(flags); 5398 __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS], nr_anon); 5399 __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_CACHE], nr_file); 5400 __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], nr_huge); 5401 __this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT], pgpgout); 5402 __this_cpu_add(memcg->stat->nr_page_events, nr_pages); 5403 memcg_check_events(memcg, dummy_page); 5404 local_irq_restore(flags); 5405 5406 if (!mem_cgroup_is_root(memcg)) 5407 css_put_many(&memcg->css, nr_pages); 5408 } 5409 5410 static void uncharge_list(struct list_head *page_list) 5411 { 5412 struct mem_cgroup *memcg = NULL; 5413 unsigned long nr_anon = 0; 5414 unsigned long nr_file = 0; 5415 unsigned long nr_huge = 0; 5416 unsigned long pgpgout = 0; 5417 struct list_head *next; 5418 struct page *page; 5419 5420 /* 5421 * Note that the list can be a single page->lru; hence the 5422 * do-while loop instead of a simple list_for_each_entry(). 5423 */ 5424 next = page_list->next; 5425 do { 5426 unsigned int nr_pages = 1; 5427 5428 page = list_entry(next, struct page, lru); 5429 next = page->lru.next; 5430 5431 VM_BUG_ON_PAGE(PageLRU(page), page); 5432 VM_BUG_ON_PAGE(page_count(page), page); 5433 5434 if (!page->mem_cgroup) 5435 continue; 5436 5437 /* 5438 * Nobody should be changing or seriously looking at 5439 * page->mem_cgroup at this point, we have fully 5440 * exclusive access to the page. 5441 */ 5442 5443 if (memcg != page->mem_cgroup) { 5444 if (memcg) { 5445 uncharge_batch(memcg, pgpgout, nr_anon, nr_file, 5446 nr_huge, page); 5447 pgpgout = nr_anon = nr_file = nr_huge = 0; 5448 } 5449 memcg = page->mem_cgroup; 5450 } 5451 5452 if (PageTransHuge(page)) { 5453 nr_pages <<= compound_order(page); 5454 VM_BUG_ON_PAGE(!PageTransHuge(page), page); 5455 nr_huge += nr_pages; 5456 } 5457 5458 if (PageAnon(page)) 5459 nr_anon += nr_pages; 5460 else 5461 nr_file += nr_pages; 5462 5463 page->mem_cgroup = NULL; 5464 5465 pgpgout++; 5466 } while (next != page_list); 5467 5468 if (memcg) 5469 uncharge_batch(memcg, pgpgout, nr_anon, nr_file, 5470 nr_huge, page); 5471 } 5472 5473 /** 5474 * mem_cgroup_uncharge - uncharge a page 5475 * @page: page to uncharge 5476 * 5477 * Uncharge a page previously charged with mem_cgroup_try_charge() and 5478 * mem_cgroup_commit_charge(). 5479 */ 5480 void mem_cgroup_uncharge(struct page *page) 5481 { 5482 if (mem_cgroup_disabled()) 5483 return; 5484 5485 /* Don't touch page->lru of any random page, pre-check: */ 5486 if (!page->mem_cgroup) 5487 return; 5488 5489 INIT_LIST_HEAD(&page->lru); 5490 uncharge_list(&page->lru); 5491 } 5492 5493 /** 5494 * mem_cgroup_uncharge_list - uncharge a list of page 5495 * @page_list: list of pages to uncharge 5496 * 5497 * Uncharge a list of pages previously charged with 5498 * mem_cgroup_try_charge() and mem_cgroup_commit_charge(). 5499 */ 5500 void mem_cgroup_uncharge_list(struct list_head *page_list) 5501 { 5502 if (mem_cgroup_disabled()) 5503 return; 5504 5505 if (!list_empty(page_list)) 5506 uncharge_list(page_list); 5507 } 5508 5509 /** 5510 * mem_cgroup_migrate - charge a page's replacement 5511 * @oldpage: currently circulating page 5512 * @newpage: replacement page 5513 * 5514 * Charge @newpage as a replacement page for @oldpage. @oldpage will 5515 * be uncharged upon free. 5516 * 5517 * Both pages must be locked, @newpage->mapping must be set up. 5518 */ 5519 void mem_cgroup_migrate(struct page *oldpage, struct page *newpage) 5520 { 5521 struct mem_cgroup *memcg; 5522 unsigned int nr_pages; 5523 bool compound; 5524 5525 VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage); 5526 VM_BUG_ON_PAGE(!PageLocked(newpage), newpage); 5527 VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage); 5528 VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage), 5529 newpage); 5530 5531 if (mem_cgroup_disabled()) 5532 return; 5533 5534 /* Page cache replacement: new page already charged? */ 5535 if (newpage->mem_cgroup) 5536 return; 5537 5538 /* Swapcache readahead pages can get replaced before being charged */ 5539 memcg = oldpage->mem_cgroup; 5540 if (!memcg) 5541 return; 5542 5543 /* Force-charge the new page. The old one will be freed soon */ 5544 compound = PageTransHuge(newpage); 5545 nr_pages = compound ? hpage_nr_pages(newpage) : 1; 5546 5547 page_counter_charge(&memcg->memory, nr_pages); 5548 if (do_memsw_account()) 5549 page_counter_charge(&memcg->memsw, nr_pages); 5550 css_get_many(&memcg->css, nr_pages); 5551 5552 commit_charge(newpage, memcg, false); 5553 5554 local_irq_disable(); 5555 mem_cgroup_charge_statistics(memcg, newpage, compound, nr_pages); 5556 memcg_check_events(memcg, newpage); 5557 local_irq_enable(); 5558 } 5559 5560 DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key); 5561 EXPORT_SYMBOL(memcg_sockets_enabled_key); 5562 5563 void sock_update_memcg(struct sock *sk) 5564 { 5565 struct mem_cgroup *memcg; 5566 5567 /* Socket cloning can throw us here with sk_cgrp already 5568 * filled. It won't however, necessarily happen from 5569 * process context. So the test for root memcg given 5570 * the current task's memcg won't help us in this case. 5571 * 5572 * Respecting the original socket's memcg is a better 5573 * decision in this case. 5574 */ 5575 if (sk->sk_memcg) { 5576 BUG_ON(mem_cgroup_is_root(sk->sk_memcg)); 5577 css_get(&sk->sk_memcg->css); 5578 return; 5579 } 5580 5581 rcu_read_lock(); 5582 memcg = mem_cgroup_from_task(current); 5583 if (memcg == root_mem_cgroup) 5584 goto out; 5585 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active) 5586 goto out; 5587 if (css_tryget_online(&memcg->css)) 5588 sk->sk_memcg = memcg; 5589 out: 5590 rcu_read_unlock(); 5591 } 5592 EXPORT_SYMBOL(sock_update_memcg); 5593 5594 void sock_release_memcg(struct sock *sk) 5595 { 5596 WARN_ON(!sk->sk_memcg); 5597 css_put(&sk->sk_memcg->css); 5598 } 5599 5600 /** 5601 * mem_cgroup_charge_skmem - charge socket memory 5602 * @memcg: memcg to charge 5603 * @nr_pages: number of pages to charge 5604 * 5605 * Charges @nr_pages to @memcg. Returns %true if the charge fit within 5606 * @memcg's configured limit, %false if the charge had to be forced. 5607 */ 5608 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages) 5609 { 5610 gfp_t gfp_mask = GFP_KERNEL; 5611 5612 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) { 5613 struct page_counter *fail; 5614 5615 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) { 5616 memcg->tcpmem_pressure = 0; 5617 return true; 5618 } 5619 page_counter_charge(&memcg->tcpmem, nr_pages); 5620 memcg->tcpmem_pressure = 1; 5621 return false; 5622 } 5623 5624 /* Don't block in the packet receive path */ 5625 if (in_softirq()) 5626 gfp_mask = GFP_NOWAIT; 5627 5628 this_cpu_add(memcg->stat->count[MEMCG_SOCK], nr_pages); 5629 5630 if (try_charge(memcg, gfp_mask, nr_pages) == 0) 5631 return true; 5632 5633 try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages); 5634 return false; 5635 } 5636 5637 /** 5638 * mem_cgroup_uncharge_skmem - uncharge socket memory 5639 * @memcg - memcg to uncharge 5640 * @nr_pages - number of pages to uncharge 5641 */ 5642 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages) 5643 { 5644 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) { 5645 page_counter_uncharge(&memcg->tcpmem, nr_pages); 5646 return; 5647 } 5648 5649 this_cpu_sub(memcg->stat->count[MEMCG_SOCK], nr_pages); 5650 5651 page_counter_uncharge(&memcg->memory, nr_pages); 5652 css_put_many(&memcg->css, nr_pages); 5653 } 5654 5655 static int __init cgroup_memory(char *s) 5656 { 5657 char *token; 5658 5659 while ((token = strsep(&s, ",")) != NULL) { 5660 if (!*token) 5661 continue; 5662 if (!strcmp(token, "nosocket")) 5663 cgroup_memory_nosocket = true; 5664 if (!strcmp(token, "nokmem")) 5665 cgroup_memory_nokmem = true; 5666 } 5667 return 0; 5668 } 5669 __setup("cgroup.memory=", cgroup_memory); 5670 5671 /* 5672 * subsys_initcall() for memory controller. 5673 * 5674 * Some parts like hotcpu_notifier() have to be initialized from this context 5675 * because of lock dependencies (cgroup_lock -> cpu hotplug) but basically 5676 * everything that doesn't depend on a specific mem_cgroup structure should 5677 * be initialized from here. 5678 */ 5679 static int __init mem_cgroup_init(void) 5680 { 5681 int cpu, node; 5682 5683 hotcpu_notifier(memcg_cpu_hotplug_callback, 0); 5684 5685 for_each_possible_cpu(cpu) 5686 INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work, 5687 drain_local_stock); 5688 5689 for_each_node(node) { 5690 struct mem_cgroup_tree_per_node *rtpn; 5691 int zone; 5692 5693 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, 5694 node_online(node) ? node : NUMA_NO_NODE); 5695 5696 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 5697 struct mem_cgroup_tree_per_zone *rtpz; 5698 5699 rtpz = &rtpn->rb_tree_per_zone[zone]; 5700 rtpz->rb_root = RB_ROOT; 5701 spin_lock_init(&rtpz->lock); 5702 } 5703 soft_limit_tree.rb_tree_per_node[node] = rtpn; 5704 } 5705 5706 return 0; 5707 } 5708 subsys_initcall(mem_cgroup_init); 5709 5710 #ifdef CONFIG_MEMCG_SWAP 5711 /** 5712 * mem_cgroup_swapout - transfer a memsw charge to swap 5713 * @page: page whose memsw charge to transfer 5714 * @entry: swap entry to move the charge to 5715 * 5716 * Transfer the memsw charge of @page to @entry. 5717 */ 5718 void mem_cgroup_swapout(struct page *page, swp_entry_t entry) 5719 { 5720 struct mem_cgroup *memcg; 5721 unsigned short oldid; 5722 5723 VM_BUG_ON_PAGE(PageLRU(page), page); 5724 VM_BUG_ON_PAGE(page_count(page), page); 5725 5726 if (!do_memsw_account()) 5727 return; 5728 5729 memcg = page->mem_cgroup; 5730 5731 /* Readahead page, never charged */ 5732 if (!memcg) 5733 return; 5734 5735 oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg)); 5736 VM_BUG_ON_PAGE(oldid, page); 5737 mem_cgroup_swap_statistics(memcg, true); 5738 5739 page->mem_cgroup = NULL; 5740 5741 if (!mem_cgroup_is_root(memcg)) 5742 page_counter_uncharge(&memcg->memory, 1); 5743 5744 /* 5745 * Interrupts should be disabled here because the caller holds the 5746 * mapping->tree_lock lock which is taken with interrupts-off. It is 5747 * important here to have the interrupts disabled because it is the 5748 * only synchronisation we have for udpating the per-CPU variables. 5749 */ 5750 VM_BUG_ON(!irqs_disabled()); 5751 mem_cgroup_charge_statistics(memcg, page, false, -1); 5752 memcg_check_events(memcg, page); 5753 } 5754 5755 /* 5756 * mem_cgroup_try_charge_swap - try charging a swap entry 5757 * @page: page being added to swap 5758 * @entry: swap entry to charge 5759 * 5760 * Try to charge @entry to the memcg that @page belongs to. 5761 * 5762 * Returns 0 on success, -ENOMEM on failure. 5763 */ 5764 int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry) 5765 { 5766 struct mem_cgroup *memcg; 5767 struct page_counter *counter; 5768 unsigned short oldid; 5769 5770 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) || !do_swap_account) 5771 return 0; 5772 5773 memcg = page->mem_cgroup; 5774 5775 /* Readahead page, never charged */ 5776 if (!memcg) 5777 return 0; 5778 5779 if (!mem_cgroup_is_root(memcg) && 5780 !page_counter_try_charge(&memcg->swap, 1, &counter)) 5781 return -ENOMEM; 5782 5783 oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg)); 5784 VM_BUG_ON_PAGE(oldid, page); 5785 mem_cgroup_swap_statistics(memcg, true); 5786 5787 css_get(&memcg->css); 5788 return 0; 5789 } 5790 5791 /** 5792 * mem_cgroup_uncharge_swap - uncharge a swap entry 5793 * @entry: swap entry to uncharge 5794 * 5795 * Drop the swap charge associated with @entry. 5796 */ 5797 void mem_cgroup_uncharge_swap(swp_entry_t entry) 5798 { 5799 struct mem_cgroup *memcg; 5800 unsigned short id; 5801 5802 if (!do_swap_account) 5803 return; 5804 5805 id = swap_cgroup_record(entry, 0); 5806 rcu_read_lock(); 5807 memcg = mem_cgroup_from_id(id); 5808 if (memcg) { 5809 if (!mem_cgroup_is_root(memcg)) { 5810 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 5811 page_counter_uncharge(&memcg->swap, 1); 5812 else 5813 page_counter_uncharge(&memcg->memsw, 1); 5814 } 5815 mem_cgroup_swap_statistics(memcg, false); 5816 css_put(&memcg->css); 5817 } 5818 rcu_read_unlock(); 5819 } 5820 5821 long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg) 5822 { 5823 long nr_swap_pages = get_nr_swap_pages(); 5824 5825 if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys)) 5826 return nr_swap_pages; 5827 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) 5828 nr_swap_pages = min_t(long, nr_swap_pages, 5829 READ_ONCE(memcg->swap.limit) - 5830 page_counter_read(&memcg->swap)); 5831 return nr_swap_pages; 5832 } 5833 5834 bool mem_cgroup_swap_full(struct page *page) 5835 { 5836 struct mem_cgroup *memcg; 5837 5838 VM_BUG_ON_PAGE(!PageLocked(page), page); 5839 5840 if (vm_swap_full()) 5841 return true; 5842 if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys)) 5843 return false; 5844 5845 memcg = page->mem_cgroup; 5846 if (!memcg) 5847 return false; 5848 5849 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) 5850 if (page_counter_read(&memcg->swap) * 2 >= memcg->swap.limit) 5851 return true; 5852 5853 return false; 5854 } 5855 5856 /* for remember boot option*/ 5857 #ifdef CONFIG_MEMCG_SWAP_ENABLED 5858 static int really_do_swap_account __initdata = 1; 5859 #else 5860 static int really_do_swap_account __initdata; 5861 #endif 5862 5863 static int __init enable_swap_account(char *s) 5864 { 5865 if (!strcmp(s, "1")) 5866 really_do_swap_account = 1; 5867 else if (!strcmp(s, "0")) 5868 really_do_swap_account = 0; 5869 return 1; 5870 } 5871 __setup("swapaccount=", enable_swap_account); 5872 5873 static u64 swap_current_read(struct cgroup_subsys_state *css, 5874 struct cftype *cft) 5875 { 5876 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5877 5878 return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE; 5879 } 5880 5881 static int swap_max_show(struct seq_file *m, void *v) 5882 { 5883 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 5884 unsigned long max = READ_ONCE(memcg->swap.limit); 5885 5886 if (max == PAGE_COUNTER_MAX) 5887 seq_puts(m, "max\n"); 5888 else 5889 seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE); 5890 5891 return 0; 5892 } 5893 5894 static ssize_t swap_max_write(struct kernfs_open_file *of, 5895 char *buf, size_t nbytes, loff_t off) 5896 { 5897 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 5898 unsigned long max; 5899 int err; 5900 5901 buf = strstrip(buf); 5902 err = page_counter_memparse(buf, "max", &max); 5903 if (err) 5904 return err; 5905 5906 mutex_lock(&memcg_limit_mutex); 5907 err = page_counter_limit(&memcg->swap, max); 5908 mutex_unlock(&memcg_limit_mutex); 5909 if (err) 5910 return err; 5911 5912 return nbytes; 5913 } 5914 5915 static struct cftype swap_files[] = { 5916 { 5917 .name = "swap.current", 5918 .flags = CFTYPE_NOT_ON_ROOT, 5919 .read_u64 = swap_current_read, 5920 }, 5921 { 5922 .name = "swap.max", 5923 .flags = CFTYPE_NOT_ON_ROOT, 5924 .seq_show = swap_max_show, 5925 .write = swap_max_write, 5926 }, 5927 { } /* terminate */ 5928 }; 5929 5930 static struct cftype memsw_cgroup_files[] = { 5931 { 5932 .name = "memsw.usage_in_bytes", 5933 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE), 5934 .read_u64 = mem_cgroup_read_u64, 5935 }, 5936 { 5937 .name = "memsw.max_usage_in_bytes", 5938 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE), 5939 .write = mem_cgroup_reset, 5940 .read_u64 = mem_cgroup_read_u64, 5941 }, 5942 { 5943 .name = "memsw.limit_in_bytes", 5944 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT), 5945 .write = mem_cgroup_write, 5946 .read_u64 = mem_cgroup_read_u64, 5947 }, 5948 { 5949 .name = "memsw.failcnt", 5950 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT), 5951 .write = mem_cgroup_reset, 5952 .read_u64 = mem_cgroup_read_u64, 5953 }, 5954 { }, /* terminate */ 5955 }; 5956 5957 static int __init mem_cgroup_swap_init(void) 5958 { 5959 if (!mem_cgroup_disabled() && really_do_swap_account) { 5960 do_swap_account = 1; 5961 WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, 5962 swap_files)); 5963 WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys, 5964 memsw_cgroup_files)); 5965 } 5966 return 0; 5967 } 5968 subsys_initcall(mem_cgroup_swap_init); 5969 5970 #endif /* CONFIG_MEMCG_SWAP */ 5971