1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* memcontrol.c - Memory Controller 3 * 4 * Copyright IBM Corporation, 2007 5 * Author Balbir Singh <balbir@linux.vnet.ibm.com> 6 * 7 * Copyright 2007 OpenVZ SWsoft Inc 8 * Author: Pavel Emelianov <xemul@openvz.org> 9 * 10 * Memory thresholds 11 * Copyright (C) 2009 Nokia Corporation 12 * Author: Kirill A. Shutemov 13 * 14 * Kernel Memory Controller 15 * Copyright (C) 2012 Parallels Inc. and Google Inc. 16 * Authors: Glauber Costa and Suleiman Souhlal 17 * 18 * Native page reclaim 19 * Charge lifetime sanitation 20 * Lockless page tracking & accounting 21 * Unified hierarchy configuration model 22 * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner 23 * 24 * Per memcg lru locking 25 * Copyright (C) 2020 Alibaba, Inc, Alex Shi 26 */ 27 28 #include <linux/page_counter.h> 29 #include <linux/memcontrol.h> 30 #include <linux/cgroup.h> 31 #include <linux/pagewalk.h> 32 #include <linux/sched/mm.h> 33 #include <linux/shmem_fs.h> 34 #include <linux/hugetlb.h> 35 #include <linux/pagemap.h> 36 #include <linux/vm_event_item.h> 37 #include <linux/smp.h> 38 #include <linux/page-flags.h> 39 #include <linux/backing-dev.h> 40 #include <linux/bit_spinlock.h> 41 #include <linux/rcupdate.h> 42 #include <linux/limits.h> 43 #include <linux/export.h> 44 #include <linux/mutex.h> 45 #include <linux/rbtree.h> 46 #include <linux/slab.h> 47 #include <linux/swap.h> 48 #include <linux/swapops.h> 49 #include <linux/spinlock.h> 50 #include <linux/eventfd.h> 51 #include <linux/poll.h> 52 #include <linux/sort.h> 53 #include <linux/fs.h> 54 #include <linux/seq_file.h> 55 #include <linux/vmpressure.h> 56 #include <linux/memremap.h> 57 #include <linux/mm_inline.h> 58 #include <linux/swap_cgroup.h> 59 #include <linux/cpu.h> 60 #include <linux/oom.h> 61 #include <linux/lockdep.h> 62 #include <linux/file.h> 63 #include <linux/resume_user_mode.h> 64 #include <linux/psi.h> 65 #include <linux/seq_buf.h> 66 #include "internal.h" 67 #include <net/sock.h> 68 #include <net/ip.h> 69 #include "slab.h" 70 #include "swap.h" 71 72 #include <linux/uaccess.h> 73 74 #include <trace/events/vmscan.h> 75 76 struct cgroup_subsys memory_cgrp_subsys __read_mostly; 77 EXPORT_SYMBOL(memory_cgrp_subsys); 78 79 struct mem_cgroup *root_mem_cgroup __read_mostly; 80 81 /* Active memory cgroup to use from an interrupt context */ 82 DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg); 83 EXPORT_PER_CPU_SYMBOL_GPL(int_active_memcg); 84 85 /* Socket memory accounting disabled? */ 86 static bool cgroup_memory_nosocket __ro_after_init; 87 88 /* Kernel memory accounting disabled? */ 89 static bool cgroup_memory_nokmem __ro_after_init; 90 91 #ifdef CONFIG_CGROUP_WRITEBACK 92 static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq); 93 #endif 94 95 /* Whether legacy memory+swap accounting is active */ 96 static bool do_memsw_account(void) 97 { 98 return !cgroup_subsys_on_dfl(memory_cgrp_subsys); 99 } 100 101 #define THRESHOLDS_EVENTS_TARGET 128 102 #define SOFTLIMIT_EVENTS_TARGET 1024 103 104 /* 105 * Cgroups above their limits are maintained in a RB-Tree, independent of 106 * their hierarchy representation 107 */ 108 109 struct mem_cgroup_tree_per_node { 110 struct rb_root rb_root; 111 struct rb_node *rb_rightmost; 112 spinlock_t lock; 113 }; 114 115 struct mem_cgroup_tree { 116 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES]; 117 }; 118 119 static struct mem_cgroup_tree soft_limit_tree __read_mostly; 120 121 /* for OOM */ 122 struct mem_cgroup_eventfd_list { 123 struct list_head list; 124 struct eventfd_ctx *eventfd; 125 }; 126 127 /* 128 * cgroup_event represents events which userspace want to receive. 129 */ 130 struct mem_cgroup_event { 131 /* 132 * memcg which the event belongs to. 133 */ 134 struct mem_cgroup *memcg; 135 /* 136 * eventfd to signal userspace about the event. 137 */ 138 struct eventfd_ctx *eventfd; 139 /* 140 * Each of these stored in a list by the cgroup. 141 */ 142 struct list_head list; 143 /* 144 * register_event() callback will be used to add new userspace 145 * waiter for changes related to this event. Use eventfd_signal() 146 * on eventfd to send notification to userspace. 147 */ 148 int (*register_event)(struct mem_cgroup *memcg, 149 struct eventfd_ctx *eventfd, const char *args); 150 /* 151 * unregister_event() callback will be called when userspace closes 152 * the eventfd or on cgroup removing. This callback must be set, 153 * if you want provide notification functionality. 154 */ 155 void (*unregister_event)(struct mem_cgroup *memcg, 156 struct eventfd_ctx *eventfd); 157 /* 158 * All fields below needed to unregister event when 159 * userspace closes eventfd. 160 */ 161 poll_table pt; 162 wait_queue_head_t *wqh; 163 wait_queue_entry_t wait; 164 struct work_struct remove; 165 }; 166 167 static void mem_cgroup_threshold(struct mem_cgroup *memcg); 168 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg); 169 170 /* Stuffs for move charges at task migration. */ 171 /* 172 * Types of charges to be moved. 173 */ 174 #define MOVE_ANON 0x1U 175 #define MOVE_FILE 0x2U 176 #define MOVE_MASK (MOVE_ANON | MOVE_FILE) 177 178 /* "mc" and its members are protected by cgroup_mutex */ 179 static struct move_charge_struct { 180 spinlock_t lock; /* for from, to */ 181 struct mm_struct *mm; 182 struct mem_cgroup *from; 183 struct mem_cgroup *to; 184 unsigned long flags; 185 unsigned long precharge; 186 unsigned long moved_charge; 187 unsigned long moved_swap; 188 struct task_struct *moving_task; /* a task moving charges */ 189 wait_queue_head_t waitq; /* a waitq for other context */ 190 } mc = { 191 .lock = __SPIN_LOCK_UNLOCKED(mc.lock), 192 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq), 193 }; 194 195 /* 196 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft 197 * limit reclaim to prevent infinite loops, if they ever occur. 198 */ 199 #define MEM_CGROUP_MAX_RECLAIM_LOOPS 100 200 #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2 201 202 /* for encoding cft->private value on file */ 203 enum res_type { 204 _MEM, 205 _MEMSWAP, 206 _KMEM, 207 _TCP, 208 }; 209 210 #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val)) 211 #define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff) 212 #define MEMFILE_ATTR(val) ((val) & 0xffff) 213 214 /* 215 * Iteration constructs for visiting all cgroups (under a tree). If 216 * loops are exited prematurely (break), mem_cgroup_iter_break() must 217 * be used for reference counting. 218 */ 219 #define for_each_mem_cgroup_tree(iter, root) \ 220 for (iter = mem_cgroup_iter(root, NULL, NULL); \ 221 iter != NULL; \ 222 iter = mem_cgroup_iter(root, iter, NULL)) 223 224 #define for_each_mem_cgroup(iter) \ 225 for (iter = mem_cgroup_iter(NULL, NULL, NULL); \ 226 iter != NULL; \ 227 iter = mem_cgroup_iter(NULL, iter, NULL)) 228 229 static inline bool task_is_dying(void) 230 { 231 return tsk_is_oom_victim(current) || fatal_signal_pending(current) || 232 (current->flags & PF_EXITING); 233 } 234 235 /* Some nice accessors for the vmpressure. */ 236 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg) 237 { 238 if (!memcg) 239 memcg = root_mem_cgroup; 240 return &memcg->vmpressure; 241 } 242 243 struct mem_cgroup *vmpressure_to_memcg(struct vmpressure *vmpr) 244 { 245 return container_of(vmpr, struct mem_cgroup, vmpressure); 246 } 247 248 #ifdef CONFIG_MEMCG_KMEM 249 static DEFINE_SPINLOCK(objcg_lock); 250 251 bool mem_cgroup_kmem_disabled(void) 252 { 253 return cgroup_memory_nokmem; 254 } 255 256 static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg, 257 unsigned int nr_pages); 258 259 static void obj_cgroup_release(struct percpu_ref *ref) 260 { 261 struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt); 262 unsigned int nr_bytes; 263 unsigned int nr_pages; 264 unsigned long flags; 265 266 /* 267 * At this point all allocated objects are freed, and 268 * objcg->nr_charged_bytes can't have an arbitrary byte value. 269 * However, it can be PAGE_SIZE or (x * PAGE_SIZE). 270 * 271 * The following sequence can lead to it: 272 * 1) CPU0: objcg == stock->cached_objcg 273 * 2) CPU1: we do a small allocation (e.g. 92 bytes), 274 * PAGE_SIZE bytes are charged 275 * 3) CPU1: a process from another memcg is allocating something, 276 * the stock if flushed, 277 * objcg->nr_charged_bytes = PAGE_SIZE - 92 278 * 5) CPU0: we do release this object, 279 * 92 bytes are added to stock->nr_bytes 280 * 6) CPU0: stock is flushed, 281 * 92 bytes are added to objcg->nr_charged_bytes 282 * 283 * In the result, nr_charged_bytes == PAGE_SIZE. 284 * This page will be uncharged in obj_cgroup_release(). 285 */ 286 nr_bytes = atomic_read(&objcg->nr_charged_bytes); 287 WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1)); 288 nr_pages = nr_bytes >> PAGE_SHIFT; 289 290 if (nr_pages) 291 obj_cgroup_uncharge_pages(objcg, nr_pages); 292 293 spin_lock_irqsave(&objcg_lock, flags); 294 list_del(&objcg->list); 295 spin_unlock_irqrestore(&objcg_lock, flags); 296 297 percpu_ref_exit(ref); 298 kfree_rcu(objcg, rcu); 299 } 300 301 static struct obj_cgroup *obj_cgroup_alloc(void) 302 { 303 struct obj_cgroup *objcg; 304 int ret; 305 306 objcg = kzalloc(sizeof(struct obj_cgroup), GFP_KERNEL); 307 if (!objcg) 308 return NULL; 309 310 ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0, 311 GFP_KERNEL); 312 if (ret) { 313 kfree(objcg); 314 return NULL; 315 } 316 INIT_LIST_HEAD(&objcg->list); 317 return objcg; 318 } 319 320 static void memcg_reparent_objcgs(struct mem_cgroup *memcg, 321 struct mem_cgroup *parent) 322 { 323 struct obj_cgroup *objcg, *iter; 324 325 objcg = rcu_replace_pointer(memcg->objcg, NULL, true); 326 327 spin_lock_irq(&objcg_lock); 328 329 /* 1) Ready to reparent active objcg. */ 330 list_add(&objcg->list, &memcg->objcg_list); 331 /* 2) Reparent active objcg and already reparented objcgs to parent. */ 332 list_for_each_entry(iter, &memcg->objcg_list, list) 333 WRITE_ONCE(iter->memcg, parent); 334 /* 3) Move already reparented objcgs to the parent's list */ 335 list_splice(&memcg->objcg_list, &parent->objcg_list); 336 337 spin_unlock_irq(&objcg_lock); 338 339 percpu_ref_kill(&objcg->refcnt); 340 } 341 342 /* 343 * A lot of the calls to the cache allocation functions are expected to be 344 * inlined by the compiler. Since the calls to memcg_slab_pre_alloc_hook() are 345 * conditional to this static branch, we'll have to allow modules that does 346 * kmem_cache_alloc and the such to see this symbol as well 347 */ 348 DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key); 349 EXPORT_SYMBOL(memcg_kmem_enabled_key); 350 #endif 351 352 /** 353 * mem_cgroup_css_from_page - css of the memcg associated with a page 354 * @page: page of interest 355 * 356 * If memcg is bound to the default hierarchy, css of the memcg associated 357 * with @page is returned. The returned css remains associated with @page 358 * until it is released. 359 * 360 * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup 361 * is returned. 362 */ 363 struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page) 364 { 365 struct mem_cgroup *memcg; 366 367 memcg = page_memcg(page); 368 369 if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys)) 370 memcg = root_mem_cgroup; 371 372 return &memcg->css; 373 } 374 375 /** 376 * page_cgroup_ino - return inode number of the memcg a page is charged to 377 * @page: the page 378 * 379 * Look up the closest online ancestor of the memory cgroup @page is charged to 380 * and return its inode number or 0 if @page is not charged to any cgroup. It 381 * is safe to call this function without holding a reference to @page. 382 * 383 * Note, this function is inherently racy, because there is nothing to prevent 384 * the cgroup inode from getting torn down and potentially reallocated a moment 385 * after page_cgroup_ino() returns, so it only should be used by callers that 386 * do not care (such as procfs interfaces). 387 */ 388 ino_t page_cgroup_ino(struct page *page) 389 { 390 struct mem_cgroup *memcg; 391 unsigned long ino = 0; 392 393 rcu_read_lock(); 394 memcg = page_memcg_check(page); 395 396 while (memcg && !(memcg->css.flags & CSS_ONLINE)) 397 memcg = parent_mem_cgroup(memcg); 398 if (memcg) 399 ino = cgroup_ino(memcg->css.cgroup); 400 rcu_read_unlock(); 401 return ino; 402 } 403 404 static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz, 405 struct mem_cgroup_tree_per_node *mctz, 406 unsigned long new_usage_in_excess) 407 { 408 struct rb_node **p = &mctz->rb_root.rb_node; 409 struct rb_node *parent = NULL; 410 struct mem_cgroup_per_node *mz_node; 411 bool rightmost = true; 412 413 if (mz->on_tree) 414 return; 415 416 mz->usage_in_excess = new_usage_in_excess; 417 if (!mz->usage_in_excess) 418 return; 419 while (*p) { 420 parent = *p; 421 mz_node = rb_entry(parent, struct mem_cgroup_per_node, 422 tree_node); 423 if (mz->usage_in_excess < mz_node->usage_in_excess) { 424 p = &(*p)->rb_left; 425 rightmost = false; 426 } else { 427 p = &(*p)->rb_right; 428 } 429 } 430 431 if (rightmost) 432 mctz->rb_rightmost = &mz->tree_node; 433 434 rb_link_node(&mz->tree_node, parent, p); 435 rb_insert_color(&mz->tree_node, &mctz->rb_root); 436 mz->on_tree = true; 437 } 438 439 static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz, 440 struct mem_cgroup_tree_per_node *mctz) 441 { 442 if (!mz->on_tree) 443 return; 444 445 if (&mz->tree_node == mctz->rb_rightmost) 446 mctz->rb_rightmost = rb_prev(&mz->tree_node); 447 448 rb_erase(&mz->tree_node, &mctz->rb_root); 449 mz->on_tree = false; 450 } 451 452 static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz, 453 struct mem_cgroup_tree_per_node *mctz) 454 { 455 unsigned long flags; 456 457 spin_lock_irqsave(&mctz->lock, flags); 458 __mem_cgroup_remove_exceeded(mz, mctz); 459 spin_unlock_irqrestore(&mctz->lock, flags); 460 } 461 462 static unsigned long soft_limit_excess(struct mem_cgroup *memcg) 463 { 464 unsigned long nr_pages = page_counter_read(&memcg->memory); 465 unsigned long soft_limit = READ_ONCE(memcg->soft_limit); 466 unsigned long excess = 0; 467 468 if (nr_pages > soft_limit) 469 excess = nr_pages - soft_limit; 470 471 return excess; 472 } 473 474 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, int nid) 475 { 476 unsigned long excess; 477 struct mem_cgroup_per_node *mz; 478 struct mem_cgroup_tree_per_node *mctz; 479 480 mctz = soft_limit_tree.rb_tree_per_node[nid]; 481 if (!mctz) 482 return; 483 /* 484 * Necessary to update all ancestors when hierarchy is used. 485 * because their event counter is not touched. 486 */ 487 for (; memcg; memcg = parent_mem_cgroup(memcg)) { 488 mz = memcg->nodeinfo[nid]; 489 excess = soft_limit_excess(memcg); 490 /* 491 * We have to update the tree if mz is on RB-tree or 492 * mem is over its softlimit. 493 */ 494 if (excess || mz->on_tree) { 495 unsigned long flags; 496 497 spin_lock_irqsave(&mctz->lock, flags); 498 /* if on-tree, remove it */ 499 if (mz->on_tree) 500 __mem_cgroup_remove_exceeded(mz, mctz); 501 /* 502 * Insert again. mz->usage_in_excess will be updated. 503 * If excess is 0, no tree ops. 504 */ 505 __mem_cgroup_insert_exceeded(mz, mctz, excess); 506 spin_unlock_irqrestore(&mctz->lock, flags); 507 } 508 } 509 } 510 511 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg) 512 { 513 struct mem_cgroup_tree_per_node *mctz; 514 struct mem_cgroup_per_node *mz; 515 int nid; 516 517 for_each_node(nid) { 518 mz = memcg->nodeinfo[nid]; 519 mctz = soft_limit_tree.rb_tree_per_node[nid]; 520 if (mctz) 521 mem_cgroup_remove_exceeded(mz, mctz); 522 } 523 } 524 525 static struct mem_cgroup_per_node * 526 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz) 527 { 528 struct mem_cgroup_per_node *mz; 529 530 retry: 531 mz = NULL; 532 if (!mctz->rb_rightmost) 533 goto done; /* Nothing to reclaim from */ 534 535 mz = rb_entry(mctz->rb_rightmost, 536 struct mem_cgroup_per_node, tree_node); 537 /* 538 * Remove the node now but someone else can add it back, 539 * we will to add it back at the end of reclaim to its correct 540 * position in the tree. 541 */ 542 __mem_cgroup_remove_exceeded(mz, mctz); 543 if (!soft_limit_excess(mz->memcg) || 544 !css_tryget(&mz->memcg->css)) 545 goto retry; 546 done: 547 return mz; 548 } 549 550 static struct mem_cgroup_per_node * 551 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz) 552 { 553 struct mem_cgroup_per_node *mz; 554 555 spin_lock_irq(&mctz->lock); 556 mz = __mem_cgroup_largest_soft_limit_node(mctz); 557 spin_unlock_irq(&mctz->lock); 558 return mz; 559 } 560 561 /* 562 * memcg and lruvec stats flushing 563 * 564 * Many codepaths leading to stats update or read are performance sensitive and 565 * adding stats flushing in such codepaths is not desirable. So, to optimize the 566 * flushing the kernel does: 567 * 568 * 1) Periodically and asynchronously flush the stats every 2 seconds to not let 569 * rstat update tree grow unbounded. 570 * 571 * 2) Flush the stats synchronously on reader side only when there are more than 572 * (MEMCG_CHARGE_BATCH * nr_cpus) update events. Though this optimization 573 * will let stats be out of sync by atmost (MEMCG_CHARGE_BATCH * nr_cpus) but 574 * only for 2 seconds due to (1). 575 */ 576 static void flush_memcg_stats_dwork(struct work_struct *w); 577 static DECLARE_DEFERRABLE_WORK(stats_flush_dwork, flush_memcg_stats_dwork); 578 static DEFINE_SPINLOCK(stats_flush_lock); 579 static DEFINE_PER_CPU(unsigned int, stats_updates); 580 static atomic_t stats_flush_threshold = ATOMIC_INIT(0); 581 static u64 flush_next_time; 582 583 #define FLUSH_TIME (2UL*HZ) 584 585 /* 586 * Accessors to ensure that preemption is disabled on PREEMPT_RT because it can 587 * not rely on this as part of an acquired spinlock_t lock. These functions are 588 * never used in hardirq context on PREEMPT_RT and therefore disabling preemtion 589 * is sufficient. 590 */ 591 static void memcg_stats_lock(void) 592 { 593 preempt_disable_nested(); 594 VM_WARN_ON_IRQS_ENABLED(); 595 } 596 597 static void __memcg_stats_lock(void) 598 { 599 preempt_disable_nested(); 600 } 601 602 static void memcg_stats_unlock(void) 603 { 604 preempt_enable_nested(); 605 } 606 607 static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val) 608 { 609 unsigned int x; 610 611 cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id()); 612 613 x = __this_cpu_add_return(stats_updates, abs(val)); 614 if (x > MEMCG_CHARGE_BATCH) { 615 /* 616 * If stats_flush_threshold exceeds the threshold 617 * (>num_online_cpus()), cgroup stats update will be triggered 618 * in __mem_cgroup_flush_stats(). Increasing this var further 619 * is redundant and simply adds overhead in atomic update. 620 */ 621 if (atomic_read(&stats_flush_threshold) <= num_online_cpus()) 622 atomic_add(x / MEMCG_CHARGE_BATCH, &stats_flush_threshold); 623 __this_cpu_write(stats_updates, 0); 624 } 625 } 626 627 static void __mem_cgroup_flush_stats(void) 628 { 629 unsigned long flag; 630 631 if (!spin_trylock_irqsave(&stats_flush_lock, flag)) 632 return; 633 634 flush_next_time = jiffies_64 + 2*FLUSH_TIME; 635 cgroup_rstat_flush_irqsafe(root_mem_cgroup->css.cgroup); 636 atomic_set(&stats_flush_threshold, 0); 637 spin_unlock_irqrestore(&stats_flush_lock, flag); 638 } 639 640 void mem_cgroup_flush_stats(void) 641 { 642 if (atomic_read(&stats_flush_threshold) > num_online_cpus()) 643 __mem_cgroup_flush_stats(); 644 } 645 646 void mem_cgroup_flush_stats_delayed(void) 647 { 648 if (time_after64(jiffies_64, flush_next_time)) 649 mem_cgroup_flush_stats(); 650 } 651 652 static void flush_memcg_stats_dwork(struct work_struct *w) 653 { 654 __mem_cgroup_flush_stats(); 655 queue_delayed_work(system_unbound_wq, &stats_flush_dwork, FLUSH_TIME); 656 } 657 658 /* Subset of vm_event_item to report for memcg event stats */ 659 static const unsigned int memcg_vm_event_stat[] = { 660 PGPGIN, 661 PGPGOUT, 662 PGSCAN_KSWAPD, 663 PGSCAN_DIRECT, 664 PGSTEAL_KSWAPD, 665 PGSTEAL_DIRECT, 666 PGFAULT, 667 PGMAJFAULT, 668 PGREFILL, 669 PGACTIVATE, 670 PGDEACTIVATE, 671 PGLAZYFREE, 672 PGLAZYFREED, 673 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP) 674 ZSWPIN, 675 ZSWPOUT, 676 #endif 677 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 678 THP_FAULT_ALLOC, 679 THP_COLLAPSE_ALLOC, 680 #endif 681 }; 682 683 #define NR_MEMCG_EVENTS ARRAY_SIZE(memcg_vm_event_stat) 684 static int mem_cgroup_events_index[NR_VM_EVENT_ITEMS] __read_mostly; 685 686 static void init_memcg_events(void) 687 { 688 int i; 689 690 for (i = 0; i < NR_MEMCG_EVENTS; ++i) 691 mem_cgroup_events_index[memcg_vm_event_stat[i]] = i + 1; 692 } 693 694 static inline int memcg_events_index(enum vm_event_item idx) 695 { 696 return mem_cgroup_events_index[idx] - 1; 697 } 698 699 struct memcg_vmstats_percpu { 700 /* Local (CPU and cgroup) page state & events */ 701 long state[MEMCG_NR_STAT]; 702 unsigned long events[NR_MEMCG_EVENTS]; 703 704 /* Delta calculation for lockless upward propagation */ 705 long state_prev[MEMCG_NR_STAT]; 706 unsigned long events_prev[NR_MEMCG_EVENTS]; 707 708 /* Cgroup1: threshold notifications & softlimit tree updates */ 709 unsigned long nr_page_events; 710 unsigned long targets[MEM_CGROUP_NTARGETS]; 711 }; 712 713 struct memcg_vmstats { 714 /* Aggregated (CPU and subtree) page state & events */ 715 long state[MEMCG_NR_STAT]; 716 unsigned long events[NR_MEMCG_EVENTS]; 717 718 /* Pending child counts during tree propagation */ 719 long state_pending[MEMCG_NR_STAT]; 720 unsigned long events_pending[NR_MEMCG_EVENTS]; 721 }; 722 723 unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx) 724 { 725 long x = READ_ONCE(memcg->vmstats->state[idx]); 726 #ifdef CONFIG_SMP 727 if (x < 0) 728 x = 0; 729 #endif 730 return x; 731 } 732 733 /** 734 * __mod_memcg_state - update cgroup memory statistics 735 * @memcg: the memory cgroup 736 * @idx: the stat item - can be enum memcg_stat_item or enum node_stat_item 737 * @val: delta to add to the counter, can be negative 738 */ 739 void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val) 740 { 741 if (mem_cgroup_disabled()) 742 return; 743 744 __this_cpu_add(memcg->vmstats_percpu->state[idx], val); 745 memcg_rstat_updated(memcg, val); 746 } 747 748 /* idx can be of type enum memcg_stat_item or node_stat_item. */ 749 static unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx) 750 { 751 long x = 0; 752 int cpu; 753 754 for_each_possible_cpu(cpu) 755 x += per_cpu(memcg->vmstats_percpu->state[idx], cpu); 756 #ifdef CONFIG_SMP 757 if (x < 0) 758 x = 0; 759 #endif 760 return x; 761 } 762 763 void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, 764 int val) 765 { 766 struct mem_cgroup_per_node *pn; 767 struct mem_cgroup *memcg; 768 769 pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); 770 memcg = pn->memcg; 771 772 /* 773 * The caller from rmap relay on disabled preemption becase they never 774 * update their counter from in-interrupt context. For these two 775 * counters we check that the update is never performed from an 776 * interrupt context while other caller need to have disabled interrupt. 777 */ 778 __memcg_stats_lock(); 779 if (IS_ENABLED(CONFIG_DEBUG_VM)) { 780 switch (idx) { 781 case NR_ANON_MAPPED: 782 case NR_FILE_MAPPED: 783 case NR_ANON_THPS: 784 case NR_SHMEM_PMDMAPPED: 785 case NR_FILE_PMDMAPPED: 786 WARN_ON_ONCE(!in_task()); 787 break; 788 default: 789 VM_WARN_ON_IRQS_ENABLED(); 790 } 791 } 792 793 /* Update memcg */ 794 __this_cpu_add(memcg->vmstats_percpu->state[idx], val); 795 796 /* Update lruvec */ 797 __this_cpu_add(pn->lruvec_stats_percpu->state[idx], val); 798 799 memcg_rstat_updated(memcg, val); 800 memcg_stats_unlock(); 801 } 802 803 /** 804 * __mod_lruvec_state - update lruvec memory statistics 805 * @lruvec: the lruvec 806 * @idx: the stat item 807 * @val: delta to add to the counter, can be negative 808 * 809 * The lruvec is the intersection of the NUMA node and a cgroup. This 810 * function updates the all three counters that are affected by a 811 * change of state at this level: per-node, per-cgroup, per-lruvec. 812 */ 813 void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, 814 int val) 815 { 816 /* Update node */ 817 __mod_node_page_state(lruvec_pgdat(lruvec), idx, val); 818 819 /* Update memcg and lruvec */ 820 if (!mem_cgroup_disabled()) 821 __mod_memcg_lruvec_state(lruvec, idx, val); 822 } 823 824 void __mod_lruvec_page_state(struct page *page, enum node_stat_item idx, 825 int val) 826 { 827 struct page *head = compound_head(page); /* rmap on tail pages */ 828 struct mem_cgroup *memcg; 829 pg_data_t *pgdat = page_pgdat(page); 830 struct lruvec *lruvec; 831 832 rcu_read_lock(); 833 memcg = page_memcg(head); 834 /* Untracked pages have no memcg, no lruvec. Update only the node */ 835 if (!memcg) { 836 rcu_read_unlock(); 837 __mod_node_page_state(pgdat, idx, val); 838 return; 839 } 840 841 lruvec = mem_cgroup_lruvec(memcg, pgdat); 842 __mod_lruvec_state(lruvec, idx, val); 843 rcu_read_unlock(); 844 } 845 EXPORT_SYMBOL(__mod_lruvec_page_state); 846 847 void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val) 848 { 849 pg_data_t *pgdat = page_pgdat(virt_to_page(p)); 850 struct mem_cgroup *memcg; 851 struct lruvec *lruvec; 852 853 rcu_read_lock(); 854 memcg = mem_cgroup_from_slab_obj(p); 855 856 /* 857 * Untracked pages have no memcg, no lruvec. Update only the 858 * node. If we reparent the slab objects to the root memcg, 859 * when we free the slab object, we need to update the per-memcg 860 * vmstats to keep it correct for the root memcg. 861 */ 862 if (!memcg) { 863 __mod_node_page_state(pgdat, idx, val); 864 } else { 865 lruvec = mem_cgroup_lruvec(memcg, pgdat); 866 __mod_lruvec_state(lruvec, idx, val); 867 } 868 rcu_read_unlock(); 869 } 870 871 /** 872 * __count_memcg_events - account VM events in a cgroup 873 * @memcg: the memory cgroup 874 * @idx: the event item 875 * @count: the number of events that occurred 876 */ 877 void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, 878 unsigned long count) 879 { 880 int index = memcg_events_index(idx); 881 882 if (mem_cgroup_disabled() || index < 0) 883 return; 884 885 memcg_stats_lock(); 886 __this_cpu_add(memcg->vmstats_percpu->events[index], count); 887 memcg_rstat_updated(memcg, count); 888 memcg_stats_unlock(); 889 } 890 891 static unsigned long memcg_events(struct mem_cgroup *memcg, int event) 892 { 893 int index = memcg_events_index(event); 894 895 if (index < 0) 896 return 0; 897 return READ_ONCE(memcg->vmstats->events[index]); 898 } 899 900 static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event) 901 { 902 long x = 0; 903 int cpu; 904 int index = memcg_events_index(event); 905 906 if (index < 0) 907 return 0; 908 909 for_each_possible_cpu(cpu) 910 x += per_cpu(memcg->vmstats_percpu->events[index], cpu); 911 return x; 912 } 913 914 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg, 915 int nr_pages) 916 { 917 /* pagein of a big page is an event. So, ignore page size */ 918 if (nr_pages > 0) 919 __count_memcg_events(memcg, PGPGIN, 1); 920 else { 921 __count_memcg_events(memcg, PGPGOUT, 1); 922 nr_pages = -nr_pages; /* for event */ 923 } 924 925 __this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages); 926 } 927 928 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg, 929 enum mem_cgroup_events_target target) 930 { 931 unsigned long val, next; 932 933 val = __this_cpu_read(memcg->vmstats_percpu->nr_page_events); 934 next = __this_cpu_read(memcg->vmstats_percpu->targets[target]); 935 /* from time_after() in jiffies.h */ 936 if ((long)(next - val) < 0) { 937 switch (target) { 938 case MEM_CGROUP_TARGET_THRESH: 939 next = val + THRESHOLDS_EVENTS_TARGET; 940 break; 941 case MEM_CGROUP_TARGET_SOFTLIMIT: 942 next = val + SOFTLIMIT_EVENTS_TARGET; 943 break; 944 default: 945 break; 946 } 947 __this_cpu_write(memcg->vmstats_percpu->targets[target], next); 948 return true; 949 } 950 return false; 951 } 952 953 /* 954 * Check events in order. 955 * 956 */ 957 static void memcg_check_events(struct mem_cgroup *memcg, int nid) 958 { 959 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 960 return; 961 962 /* threshold event is triggered in finer grain than soft limit */ 963 if (unlikely(mem_cgroup_event_ratelimit(memcg, 964 MEM_CGROUP_TARGET_THRESH))) { 965 bool do_softlimit; 966 967 do_softlimit = mem_cgroup_event_ratelimit(memcg, 968 MEM_CGROUP_TARGET_SOFTLIMIT); 969 mem_cgroup_threshold(memcg); 970 if (unlikely(do_softlimit)) 971 mem_cgroup_update_tree(memcg, nid); 972 } 973 } 974 975 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) 976 { 977 /* 978 * mm_update_next_owner() may clear mm->owner to NULL 979 * if it races with swapoff, page migration, etc. 980 * So this can be called with p == NULL. 981 */ 982 if (unlikely(!p)) 983 return NULL; 984 985 return mem_cgroup_from_css(task_css(p, memory_cgrp_id)); 986 } 987 EXPORT_SYMBOL(mem_cgroup_from_task); 988 989 static __always_inline struct mem_cgroup *active_memcg(void) 990 { 991 if (!in_task()) 992 return this_cpu_read(int_active_memcg); 993 else 994 return current->active_memcg; 995 } 996 997 /** 998 * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg. 999 * @mm: mm from which memcg should be extracted. It can be NULL. 1000 * 1001 * Obtain a reference on mm->memcg and returns it if successful. If mm 1002 * is NULL, then the memcg is chosen as follows: 1003 * 1) The active memcg, if set. 1004 * 2) current->mm->memcg, if available 1005 * 3) root memcg 1006 * If mem_cgroup is disabled, NULL is returned. 1007 */ 1008 struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm) 1009 { 1010 struct mem_cgroup *memcg; 1011 1012 if (mem_cgroup_disabled()) 1013 return NULL; 1014 1015 /* 1016 * Page cache insertions can happen without an 1017 * actual mm context, e.g. during disk probing 1018 * on boot, loopback IO, acct() writes etc. 1019 * 1020 * No need to css_get on root memcg as the reference 1021 * counting is disabled on the root level in the 1022 * cgroup core. See CSS_NO_REF. 1023 */ 1024 if (unlikely(!mm)) { 1025 memcg = active_memcg(); 1026 if (unlikely(memcg)) { 1027 /* remote memcg must hold a ref */ 1028 css_get(&memcg->css); 1029 return memcg; 1030 } 1031 mm = current->mm; 1032 if (unlikely(!mm)) 1033 return root_mem_cgroup; 1034 } 1035 1036 rcu_read_lock(); 1037 do { 1038 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); 1039 if (unlikely(!memcg)) 1040 memcg = root_mem_cgroup; 1041 } while (!css_tryget(&memcg->css)); 1042 rcu_read_unlock(); 1043 return memcg; 1044 } 1045 EXPORT_SYMBOL(get_mem_cgroup_from_mm); 1046 1047 static __always_inline bool memcg_kmem_bypass(void) 1048 { 1049 /* Allow remote memcg charging from any context. */ 1050 if (unlikely(active_memcg())) 1051 return false; 1052 1053 /* Memcg to charge can't be determined. */ 1054 if (!in_task() || !current->mm || (current->flags & PF_KTHREAD)) 1055 return true; 1056 1057 return false; 1058 } 1059 1060 /** 1061 * mem_cgroup_iter - iterate over memory cgroup hierarchy 1062 * @root: hierarchy root 1063 * @prev: previously returned memcg, NULL on first invocation 1064 * @reclaim: cookie for shared reclaim walks, NULL for full walks 1065 * 1066 * Returns references to children of the hierarchy below @root, or 1067 * @root itself, or %NULL after a full round-trip. 1068 * 1069 * Caller must pass the return value in @prev on subsequent 1070 * invocations for reference counting, or use mem_cgroup_iter_break() 1071 * to cancel a hierarchy walk before the round-trip is complete. 1072 * 1073 * Reclaimers can specify a node in @reclaim to divide up the memcgs 1074 * in the hierarchy among all concurrent reclaimers operating on the 1075 * same node. 1076 */ 1077 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root, 1078 struct mem_cgroup *prev, 1079 struct mem_cgroup_reclaim_cookie *reclaim) 1080 { 1081 struct mem_cgroup_reclaim_iter *iter; 1082 struct cgroup_subsys_state *css = NULL; 1083 struct mem_cgroup *memcg = NULL; 1084 struct mem_cgroup *pos = NULL; 1085 1086 if (mem_cgroup_disabled()) 1087 return NULL; 1088 1089 if (!root) 1090 root = root_mem_cgroup; 1091 1092 rcu_read_lock(); 1093 1094 if (reclaim) { 1095 struct mem_cgroup_per_node *mz; 1096 1097 mz = root->nodeinfo[reclaim->pgdat->node_id]; 1098 iter = &mz->iter; 1099 1100 /* 1101 * On start, join the current reclaim iteration cycle. 1102 * Exit when a concurrent walker completes it. 1103 */ 1104 if (!prev) 1105 reclaim->generation = iter->generation; 1106 else if (reclaim->generation != iter->generation) 1107 goto out_unlock; 1108 1109 while (1) { 1110 pos = READ_ONCE(iter->position); 1111 if (!pos || css_tryget(&pos->css)) 1112 break; 1113 /* 1114 * css reference reached zero, so iter->position will 1115 * be cleared by ->css_released. However, we should not 1116 * rely on this happening soon, because ->css_released 1117 * is called from a work queue, and by busy-waiting we 1118 * might block it. So we clear iter->position right 1119 * away. 1120 */ 1121 (void)cmpxchg(&iter->position, pos, NULL); 1122 } 1123 } else if (prev) { 1124 pos = prev; 1125 } 1126 1127 if (pos) 1128 css = &pos->css; 1129 1130 for (;;) { 1131 css = css_next_descendant_pre(css, &root->css); 1132 if (!css) { 1133 /* 1134 * Reclaimers share the hierarchy walk, and a 1135 * new one might jump in right at the end of 1136 * the hierarchy - make sure they see at least 1137 * one group and restart from the beginning. 1138 */ 1139 if (!prev) 1140 continue; 1141 break; 1142 } 1143 1144 /* 1145 * Verify the css and acquire a reference. The root 1146 * is provided by the caller, so we know it's alive 1147 * and kicking, and don't take an extra reference. 1148 */ 1149 if (css == &root->css || css_tryget(css)) { 1150 memcg = mem_cgroup_from_css(css); 1151 break; 1152 } 1153 } 1154 1155 if (reclaim) { 1156 /* 1157 * The position could have already been updated by a competing 1158 * thread, so check that the value hasn't changed since we read 1159 * it to avoid reclaiming from the same cgroup twice. 1160 */ 1161 (void)cmpxchg(&iter->position, pos, memcg); 1162 1163 if (pos) 1164 css_put(&pos->css); 1165 1166 if (!memcg) 1167 iter->generation++; 1168 } 1169 1170 out_unlock: 1171 rcu_read_unlock(); 1172 if (prev && prev != root) 1173 css_put(&prev->css); 1174 1175 return memcg; 1176 } 1177 1178 /** 1179 * mem_cgroup_iter_break - abort a hierarchy walk prematurely 1180 * @root: hierarchy root 1181 * @prev: last visited hierarchy member as returned by mem_cgroup_iter() 1182 */ 1183 void mem_cgroup_iter_break(struct mem_cgroup *root, 1184 struct mem_cgroup *prev) 1185 { 1186 if (!root) 1187 root = root_mem_cgroup; 1188 if (prev && prev != root) 1189 css_put(&prev->css); 1190 } 1191 1192 static void __invalidate_reclaim_iterators(struct mem_cgroup *from, 1193 struct mem_cgroup *dead_memcg) 1194 { 1195 struct mem_cgroup_reclaim_iter *iter; 1196 struct mem_cgroup_per_node *mz; 1197 int nid; 1198 1199 for_each_node(nid) { 1200 mz = from->nodeinfo[nid]; 1201 iter = &mz->iter; 1202 cmpxchg(&iter->position, dead_memcg, NULL); 1203 } 1204 } 1205 1206 static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg) 1207 { 1208 struct mem_cgroup *memcg = dead_memcg; 1209 struct mem_cgroup *last; 1210 1211 do { 1212 __invalidate_reclaim_iterators(memcg, dead_memcg); 1213 last = memcg; 1214 } while ((memcg = parent_mem_cgroup(memcg))); 1215 1216 /* 1217 * When cgroup1 non-hierarchy mode is used, 1218 * parent_mem_cgroup() does not walk all the way up to the 1219 * cgroup root (root_mem_cgroup). So we have to handle 1220 * dead_memcg from cgroup root separately. 1221 */ 1222 if (last != root_mem_cgroup) 1223 __invalidate_reclaim_iterators(root_mem_cgroup, 1224 dead_memcg); 1225 } 1226 1227 /** 1228 * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy 1229 * @memcg: hierarchy root 1230 * @fn: function to call for each task 1231 * @arg: argument passed to @fn 1232 * 1233 * This function iterates over tasks attached to @memcg or to any of its 1234 * descendants and calls @fn for each task. If @fn returns a non-zero 1235 * value, the function breaks the iteration loop and returns the value. 1236 * Otherwise, it will iterate over all tasks and return 0. 1237 * 1238 * This function must not be called for the root memory cgroup. 1239 */ 1240 int mem_cgroup_scan_tasks(struct mem_cgroup *memcg, 1241 int (*fn)(struct task_struct *, void *), void *arg) 1242 { 1243 struct mem_cgroup *iter; 1244 int ret = 0; 1245 1246 BUG_ON(memcg == root_mem_cgroup); 1247 1248 for_each_mem_cgroup_tree(iter, memcg) { 1249 struct css_task_iter it; 1250 struct task_struct *task; 1251 1252 css_task_iter_start(&iter->css, CSS_TASK_ITER_PROCS, &it); 1253 while (!ret && (task = css_task_iter_next(&it))) 1254 ret = fn(task, arg); 1255 css_task_iter_end(&it); 1256 if (ret) { 1257 mem_cgroup_iter_break(memcg, iter); 1258 break; 1259 } 1260 } 1261 return ret; 1262 } 1263 1264 #ifdef CONFIG_DEBUG_VM 1265 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio) 1266 { 1267 struct mem_cgroup *memcg; 1268 1269 if (mem_cgroup_disabled()) 1270 return; 1271 1272 memcg = folio_memcg(folio); 1273 1274 if (!memcg) 1275 VM_BUG_ON_FOLIO(lruvec_memcg(lruvec) != root_mem_cgroup, folio); 1276 else 1277 VM_BUG_ON_FOLIO(lruvec_memcg(lruvec) != memcg, folio); 1278 } 1279 #endif 1280 1281 /** 1282 * folio_lruvec_lock - Lock the lruvec for a folio. 1283 * @folio: Pointer to the folio. 1284 * 1285 * These functions are safe to use under any of the following conditions: 1286 * - folio locked 1287 * - folio_test_lru false 1288 * - folio_memcg_lock() 1289 * - folio frozen (refcount of 0) 1290 * 1291 * Return: The lruvec this folio is on with its lock held. 1292 */ 1293 struct lruvec *folio_lruvec_lock(struct folio *folio) 1294 { 1295 struct lruvec *lruvec = folio_lruvec(folio); 1296 1297 spin_lock(&lruvec->lru_lock); 1298 lruvec_memcg_debug(lruvec, folio); 1299 1300 return lruvec; 1301 } 1302 1303 /** 1304 * folio_lruvec_lock_irq - Lock the lruvec for a folio. 1305 * @folio: Pointer to the folio. 1306 * 1307 * These functions are safe to use under any of the following conditions: 1308 * - folio locked 1309 * - folio_test_lru false 1310 * - folio_memcg_lock() 1311 * - folio frozen (refcount of 0) 1312 * 1313 * Return: The lruvec this folio is on with its lock held and interrupts 1314 * disabled. 1315 */ 1316 struct lruvec *folio_lruvec_lock_irq(struct folio *folio) 1317 { 1318 struct lruvec *lruvec = folio_lruvec(folio); 1319 1320 spin_lock_irq(&lruvec->lru_lock); 1321 lruvec_memcg_debug(lruvec, folio); 1322 1323 return lruvec; 1324 } 1325 1326 /** 1327 * folio_lruvec_lock_irqsave - Lock the lruvec for a folio. 1328 * @folio: Pointer to the folio. 1329 * @flags: Pointer to irqsave flags. 1330 * 1331 * These functions are safe to use under any of the following conditions: 1332 * - folio locked 1333 * - folio_test_lru false 1334 * - folio_memcg_lock() 1335 * - folio frozen (refcount of 0) 1336 * 1337 * Return: The lruvec this folio is on with its lock held and interrupts 1338 * disabled. 1339 */ 1340 struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio, 1341 unsigned long *flags) 1342 { 1343 struct lruvec *lruvec = folio_lruvec(folio); 1344 1345 spin_lock_irqsave(&lruvec->lru_lock, *flags); 1346 lruvec_memcg_debug(lruvec, folio); 1347 1348 return lruvec; 1349 } 1350 1351 /** 1352 * mem_cgroup_update_lru_size - account for adding or removing an lru page 1353 * @lruvec: mem_cgroup per zone lru vector 1354 * @lru: index of lru list the page is sitting on 1355 * @zid: zone id of the accounted pages 1356 * @nr_pages: positive when adding or negative when removing 1357 * 1358 * This function must be called under lru_lock, just before a page is added 1359 * to or just after a page is removed from an lru list. 1360 */ 1361 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, 1362 int zid, int nr_pages) 1363 { 1364 struct mem_cgroup_per_node *mz; 1365 unsigned long *lru_size; 1366 long size; 1367 1368 if (mem_cgroup_disabled()) 1369 return; 1370 1371 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); 1372 lru_size = &mz->lru_zone_size[zid][lru]; 1373 1374 if (nr_pages < 0) 1375 *lru_size += nr_pages; 1376 1377 size = *lru_size; 1378 if (WARN_ONCE(size < 0, 1379 "%s(%p, %d, %d): lru_size %ld\n", 1380 __func__, lruvec, lru, nr_pages, size)) { 1381 VM_BUG_ON(1); 1382 *lru_size = 0; 1383 } 1384 1385 if (nr_pages > 0) 1386 *lru_size += nr_pages; 1387 } 1388 1389 /** 1390 * mem_cgroup_margin - calculate chargeable space of a memory cgroup 1391 * @memcg: the memory cgroup 1392 * 1393 * Returns the maximum amount of memory @mem can be charged with, in 1394 * pages. 1395 */ 1396 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg) 1397 { 1398 unsigned long margin = 0; 1399 unsigned long count; 1400 unsigned long limit; 1401 1402 count = page_counter_read(&memcg->memory); 1403 limit = READ_ONCE(memcg->memory.max); 1404 if (count < limit) 1405 margin = limit - count; 1406 1407 if (do_memsw_account()) { 1408 count = page_counter_read(&memcg->memsw); 1409 limit = READ_ONCE(memcg->memsw.max); 1410 if (count < limit) 1411 margin = min(margin, limit - count); 1412 else 1413 margin = 0; 1414 } 1415 1416 return margin; 1417 } 1418 1419 /* 1420 * A routine for checking "mem" is under move_account() or not. 1421 * 1422 * Checking a cgroup is mc.from or mc.to or under hierarchy of 1423 * moving cgroups. This is for waiting at high-memory pressure 1424 * caused by "move". 1425 */ 1426 static bool mem_cgroup_under_move(struct mem_cgroup *memcg) 1427 { 1428 struct mem_cgroup *from; 1429 struct mem_cgroup *to; 1430 bool ret = false; 1431 /* 1432 * Unlike task_move routines, we access mc.to, mc.from not under 1433 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead. 1434 */ 1435 spin_lock(&mc.lock); 1436 from = mc.from; 1437 to = mc.to; 1438 if (!from) 1439 goto unlock; 1440 1441 ret = mem_cgroup_is_descendant(from, memcg) || 1442 mem_cgroup_is_descendant(to, memcg); 1443 unlock: 1444 spin_unlock(&mc.lock); 1445 return ret; 1446 } 1447 1448 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg) 1449 { 1450 if (mc.moving_task && current != mc.moving_task) { 1451 if (mem_cgroup_under_move(memcg)) { 1452 DEFINE_WAIT(wait); 1453 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE); 1454 /* moving charge context might have finished. */ 1455 if (mc.moving_task) 1456 schedule(); 1457 finish_wait(&mc.waitq, &wait); 1458 return true; 1459 } 1460 } 1461 return false; 1462 } 1463 1464 struct memory_stat { 1465 const char *name; 1466 unsigned int idx; 1467 }; 1468 1469 static const struct memory_stat memory_stats[] = { 1470 { "anon", NR_ANON_MAPPED }, 1471 { "file", NR_FILE_PAGES }, 1472 { "kernel", MEMCG_KMEM }, 1473 { "kernel_stack", NR_KERNEL_STACK_KB }, 1474 { "pagetables", NR_PAGETABLE }, 1475 { "sec_pagetables", NR_SECONDARY_PAGETABLE }, 1476 { "percpu", MEMCG_PERCPU_B }, 1477 { "sock", MEMCG_SOCK }, 1478 { "vmalloc", MEMCG_VMALLOC }, 1479 { "shmem", NR_SHMEM }, 1480 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP) 1481 { "zswap", MEMCG_ZSWAP_B }, 1482 { "zswapped", MEMCG_ZSWAPPED }, 1483 #endif 1484 { "file_mapped", NR_FILE_MAPPED }, 1485 { "file_dirty", NR_FILE_DIRTY }, 1486 { "file_writeback", NR_WRITEBACK }, 1487 #ifdef CONFIG_SWAP 1488 { "swapcached", NR_SWAPCACHE }, 1489 #endif 1490 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1491 { "anon_thp", NR_ANON_THPS }, 1492 { "file_thp", NR_FILE_THPS }, 1493 { "shmem_thp", NR_SHMEM_THPS }, 1494 #endif 1495 { "inactive_anon", NR_INACTIVE_ANON }, 1496 { "active_anon", NR_ACTIVE_ANON }, 1497 { "inactive_file", NR_INACTIVE_FILE }, 1498 { "active_file", NR_ACTIVE_FILE }, 1499 { "unevictable", NR_UNEVICTABLE }, 1500 { "slab_reclaimable", NR_SLAB_RECLAIMABLE_B }, 1501 { "slab_unreclaimable", NR_SLAB_UNRECLAIMABLE_B }, 1502 1503 /* The memory events */ 1504 { "workingset_refault_anon", WORKINGSET_REFAULT_ANON }, 1505 { "workingset_refault_file", WORKINGSET_REFAULT_FILE }, 1506 { "workingset_activate_anon", WORKINGSET_ACTIVATE_ANON }, 1507 { "workingset_activate_file", WORKINGSET_ACTIVATE_FILE }, 1508 { "workingset_restore_anon", WORKINGSET_RESTORE_ANON }, 1509 { "workingset_restore_file", WORKINGSET_RESTORE_FILE }, 1510 { "workingset_nodereclaim", WORKINGSET_NODERECLAIM }, 1511 }; 1512 1513 /* Translate stat items to the correct unit for memory.stat output */ 1514 static int memcg_page_state_unit(int item) 1515 { 1516 switch (item) { 1517 case MEMCG_PERCPU_B: 1518 case MEMCG_ZSWAP_B: 1519 case NR_SLAB_RECLAIMABLE_B: 1520 case NR_SLAB_UNRECLAIMABLE_B: 1521 case WORKINGSET_REFAULT_ANON: 1522 case WORKINGSET_REFAULT_FILE: 1523 case WORKINGSET_ACTIVATE_ANON: 1524 case WORKINGSET_ACTIVATE_FILE: 1525 case WORKINGSET_RESTORE_ANON: 1526 case WORKINGSET_RESTORE_FILE: 1527 case WORKINGSET_NODERECLAIM: 1528 return 1; 1529 case NR_KERNEL_STACK_KB: 1530 return SZ_1K; 1531 default: 1532 return PAGE_SIZE; 1533 } 1534 } 1535 1536 static inline unsigned long memcg_page_state_output(struct mem_cgroup *memcg, 1537 int item) 1538 { 1539 return memcg_page_state(memcg, item) * memcg_page_state_unit(item); 1540 } 1541 1542 static void memory_stat_format(struct mem_cgroup *memcg, char *buf, int bufsize) 1543 { 1544 struct seq_buf s; 1545 int i; 1546 1547 seq_buf_init(&s, buf, bufsize); 1548 1549 /* 1550 * Provide statistics on the state of the memory subsystem as 1551 * well as cumulative event counters that show past behavior. 1552 * 1553 * This list is ordered following a combination of these gradients: 1554 * 1) generic big picture -> specifics and details 1555 * 2) reflecting userspace activity -> reflecting kernel heuristics 1556 * 1557 * Current memory state: 1558 */ 1559 mem_cgroup_flush_stats(); 1560 1561 for (i = 0; i < ARRAY_SIZE(memory_stats); i++) { 1562 u64 size; 1563 1564 size = memcg_page_state_output(memcg, memory_stats[i].idx); 1565 seq_buf_printf(&s, "%s %llu\n", memory_stats[i].name, size); 1566 1567 if (unlikely(memory_stats[i].idx == NR_SLAB_UNRECLAIMABLE_B)) { 1568 size += memcg_page_state_output(memcg, 1569 NR_SLAB_RECLAIMABLE_B); 1570 seq_buf_printf(&s, "slab %llu\n", size); 1571 } 1572 } 1573 1574 /* Accumulated memory events */ 1575 seq_buf_printf(&s, "pgscan %lu\n", 1576 memcg_events(memcg, PGSCAN_KSWAPD) + 1577 memcg_events(memcg, PGSCAN_DIRECT)); 1578 seq_buf_printf(&s, "pgsteal %lu\n", 1579 memcg_events(memcg, PGSTEAL_KSWAPD) + 1580 memcg_events(memcg, PGSTEAL_DIRECT)); 1581 1582 for (i = 0; i < ARRAY_SIZE(memcg_vm_event_stat); i++) { 1583 if (memcg_vm_event_stat[i] == PGPGIN || 1584 memcg_vm_event_stat[i] == PGPGOUT) 1585 continue; 1586 1587 seq_buf_printf(&s, "%s %lu\n", 1588 vm_event_name(memcg_vm_event_stat[i]), 1589 memcg_events(memcg, memcg_vm_event_stat[i])); 1590 } 1591 1592 /* The above should easily fit into one page */ 1593 WARN_ON_ONCE(seq_buf_has_overflowed(&s)); 1594 } 1595 1596 #define K(x) ((x) << (PAGE_SHIFT-10)) 1597 /** 1598 * mem_cgroup_print_oom_context: Print OOM information relevant to 1599 * memory controller. 1600 * @memcg: The memory cgroup that went over limit 1601 * @p: Task that is going to be killed 1602 * 1603 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is 1604 * enabled 1605 */ 1606 void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p) 1607 { 1608 rcu_read_lock(); 1609 1610 if (memcg) { 1611 pr_cont(",oom_memcg="); 1612 pr_cont_cgroup_path(memcg->css.cgroup); 1613 } else 1614 pr_cont(",global_oom"); 1615 if (p) { 1616 pr_cont(",task_memcg="); 1617 pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id)); 1618 } 1619 rcu_read_unlock(); 1620 } 1621 1622 /** 1623 * mem_cgroup_print_oom_meminfo: Print OOM memory information relevant to 1624 * memory controller. 1625 * @memcg: The memory cgroup that went over limit 1626 */ 1627 void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg) 1628 { 1629 /* Use static buffer, for the caller is holding oom_lock. */ 1630 static char buf[PAGE_SIZE]; 1631 1632 lockdep_assert_held(&oom_lock); 1633 1634 pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n", 1635 K((u64)page_counter_read(&memcg->memory)), 1636 K((u64)READ_ONCE(memcg->memory.max)), memcg->memory.failcnt); 1637 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 1638 pr_info("swap: usage %llukB, limit %llukB, failcnt %lu\n", 1639 K((u64)page_counter_read(&memcg->swap)), 1640 K((u64)READ_ONCE(memcg->swap.max)), memcg->swap.failcnt); 1641 else { 1642 pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n", 1643 K((u64)page_counter_read(&memcg->memsw)), 1644 K((u64)memcg->memsw.max), memcg->memsw.failcnt); 1645 pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n", 1646 K((u64)page_counter_read(&memcg->kmem)), 1647 K((u64)memcg->kmem.max), memcg->kmem.failcnt); 1648 } 1649 1650 pr_info("Memory cgroup stats for "); 1651 pr_cont_cgroup_path(memcg->css.cgroup); 1652 pr_cont(":"); 1653 memory_stat_format(memcg, buf, sizeof(buf)); 1654 pr_info("%s", buf); 1655 } 1656 1657 /* 1658 * Return the memory (and swap, if configured) limit for a memcg. 1659 */ 1660 unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg) 1661 { 1662 unsigned long max = READ_ONCE(memcg->memory.max); 1663 1664 if (do_memsw_account()) { 1665 if (mem_cgroup_swappiness(memcg)) { 1666 /* Calculate swap excess capacity from memsw limit */ 1667 unsigned long swap = READ_ONCE(memcg->memsw.max) - max; 1668 1669 max += min(swap, (unsigned long)total_swap_pages); 1670 } 1671 } else { 1672 if (mem_cgroup_swappiness(memcg)) 1673 max += min(READ_ONCE(memcg->swap.max), 1674 (unsigned long)total_swap_pages); 1675 } 1676 return max; 1677 } 1678 1679 unsigned long mem_cgroup_size(struct mem_cgroup *memcg) 1680 { 1681 return page_counter_read(&memcg->memory); 1682 } 1683 1684 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask, 1685 int order) 1686 { 1687 struct oom_control oc = { 1688 .zonelist = NULL, 1689 .nodemask = NULL, 1690 .memcg = memcg, 1691 .gfp_mask = gfp_mask, 1692 .order = order, 1693 }; 1694 bool ret = true; 1695 1696 if (mutex_lock_killable(&oom_lock)) 1697 return true; 1698 1699 if (mem_cgroup_margin(memcg) >= (1 << order)) 1700 goto unlock; 1701 1702 /* 1703 * A few threads which were not waiting at mutex_lock_killable() can 1704 * fail to bail out. Therefore, check again after holding oom_lock. 1705 */ 1706 ret = task_is_dying() || out_of_memory(&oc); 1707 1708 unlock: 1709 mutex_unlock(&oom_lock); 1710 return ret; 1711 } 1712 1713 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg, 1714 pg_data_t *pgdat, 1715 gfp_t gfp_mask, 1716 unsigned long *total_scanned) 1717 { 1718 struct mem_cgroup *victim = NULL; 1719 int total = 0; 1720 int loop = 0; 1721 unsigned long excess; 1722 unsigned long nr_scanned; 1723 struct mem_cgroup_reclaim_cookie reclaim = { 1724 .pgdat = pgdat, 1725 }; 1726 1727 excess = soft_limit_excess(root_memcg); 1728 1729 while (1) { 1730 victim = mem_cgroup_iter(root_memcg, victim, &reclaim); 1731 if (!victim) { 1732 loop++; 1733 if (loop >= 2) { 1734 /* 1735 * If we have not been able to reclaim 1736 * anything, it might because there are 1737 * no reclaimable pages under this hierarchy 1738 */ 1739 if (!total) 1740 break; 1741 /* 1742 * We want to do more targeted reclaim. 1743 * excess >> 2 is not to excessive so as to 1744 * reclaim too much, nor too less that we keep 1745 * coming back to reclaim from this cgroup 1746 */ 1747 if (total >= (excess >> 2) || 1748 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) 1749 break; 1750 } 1751 continue; 1752 } 1753 total += mem_cgroup_shrink_node(victim, gfp_mask, false, 1754 pgdat, &nr_scanned); 1755 *total_scanned += nr_scanned; 1756 if (!soft_limit_excess(root_memcg)) 1757 break; 1758 } 1759 mem_cgroup_iter_break(root_memcg, victim); 1760 return total; 1761 } 1762 1763 #ifdef CONFIG_LOCKDEP 1764 static struct lockdep_map memcg_oom_lock_dep_map = { 1765 .name = "memcg_oom_lock", 1766 }; 1767 #endif 1768 1769 static DEFINE_SPINLOCK(memcg_oom_lock); 1770 1771 /* 1772 * Check OOM-Killer is already running under our hierarchy. 1773 * If someone is running, return false. 1774 */ 1775 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg) 1776 { 1777 struct mem_cgroup *iter, *failed = NULL; 1778 1779 spin_lock(&memcg_oom_lock); 1780 1781 for_each_mem_cgroup_tree(iter, memcg) { 1782 if (iter->oom_lock) { 1783 /* 1784 * this subtree of our hierarchy is already locked 1785 * so we cannot give a lock. 1786 */ 1787 failed = iter; 1788 mem_cgroup_iter_break(memcg, iter); 1789 break; 1790 } else 1791 iter->oom_lock = true; 1792 } 1793 1794 if (failed) { 1795 /* 1796 * OK, we failed to lock the whole subtree so we have 1797 * to clean up what we set up to the failing subtree 1798 */ 1799 for_each_mem_cgroup_tree(iter, memcg) { 1800 if (iter == failed) { 1801 mem_cgroup_iter_break(memcg, iter); 1802 break; 1803 } 1804 iter->oom_lock = false; 1805 } 1806 } else 1807 mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_); 1808 1809 spin_unlock(&memcg_oom_lock); 1810 1811 return !failed; 1812 } 1813 1814 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg) 1815 { 1816 struct mem_cgroup *iter; 1817 1818 spin_lock(&memcg_oom_lock); 1819 mutex_release(&memcg_oom_lock_dep_map, _RET_IP_); 1820 for_each_mem_cgroup_tree(iter, memcg) 1821 iter->oom_lock = false; 1822 spin_unlock(&memcg_oom_lock); 1823 } 1824 1825 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg) 1826 { 1827 struct mem_cgroup *iter; 1828 1829 spin_lock(&memcg_oom_lock); 1830 for_each_mem_cgroup_tree(iter, memcg) 1831 iter->under_oom++; 1832 spin_unlock(&memcg_oom_lock); 1833 } 1834 1835 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg) 1836 { 1837 struct mem_cgroup *iter; 1838 1839 /* 1840 * Be careful about under_oom underflows because a child memcg 1841 * could have been added after mem_cgroup_mark_under_oom. 1842 */ 1843 spin_lock(&memcg_oom_lock); 1844 for_each_mem_cgroup_tree(iter, memcg) 1845 if (iter->under_oom > 0) 1846 iter->under_oom--; 1847 spin_unlock(&memcg_oom_lock); 1848 } 1849 1850 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq); 1851 1852 struct oom_wait_info { 1853 struct mem_cgroup *memcg; 1854 wait_queue_entry_t wait; 1855 }; 1856 1857 static int memcg_oom_wake_function(wait_queue_entry_t *wait, 1858 unsigned mode, int sync, void *arg) 1859 { 1860 struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg; 1861 struct mem_cgroup *oom_wait_memcg; 1862 struct oom_wait_info *oom_wait_info; 1863 1864 oom_wait_info = container_of(wait, struct oom_wait_info, wait); 1865 oom_wait_memcg = oom_wait_info->memcg; 1866 1867 if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) && 1868 !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg)) 1869 return 0; 1870 return autoremove_wake_function(wait, mode, sync, arg); 1871 } 1872 1873 static void memcg_oom_recover(struct mem_cgroup *memcg) 1874 { 1875 /* 1876 * For the following lockless ->under_oom test, the only required 1877 * guarantee is that it must see the state asserted by an OOM when 1878 * this function is called as a result of userland actions 1879 * triggered by the notification of the OOM. This is trivially 1880 * achieved by invoking mem_cgroup_mark_under_oom() before 1881 * triggering notification. 1882 */ 1883 if (memcg && memcg->under_oom) 1884 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg); 1885 } 1886 1887 /* 1888 * Returns true if successfully killed one or more processes. Though in some 1889 * corner cases it can return true even without killing any process. 1890 */ 1891 static bool mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order) 1892 { 1893 bool locked, ret; 1894 1895 if (order > PAGE_ALLOC_COSTLY_ORDER) 1896 return false; 1897 1898 memcg_memory_event(memcg, MEMCG_OOM); 1899 1900 /* 1901 * We are in the middle of the charge context here, so we 1902 * don't want to block when potentially sitting on a callstack 1903 * that holds all kinds of filesystem and mm locks. 1904 * 1905 * cgroup1 allows disabling the OOM killer and waiting for outside 1906 * handling until the charge can succeed; remember the context and put 1907 * the task to sleep at the end of the page fault when all locks are 1908 * released. 1909 * 1910 * On the other hand, in-kernel OOM killer allows for an async victim 1911 * memory reclaim (oom_reaper) and that means that we are not solely 1912 * relying on the oom victim to make a forward progress and we can 1913 * invoke the oom killer here. 1914 * 1915 * Please note that mem_cgroup_out_of_memory might fail to find a 1916 * victim and then we have to bail out from the charge path. 1917 */ 1918 if (memcg->oom_kill_disable) { 1919 if (current->in_user_fault) { 1920 css_get(&memcg->css); 1921 current->memcg_in_oom = memcg; 1922 current->memcg_oom_gfp_mask = mask; 1923 current->memcg_oom_order = order; 1924 } 1925 return false; 1926 } 1927 1928 mem_cgroup_mark_under_oom(memcg); 1929 1930 locked = mem_cgroup_oom_trylock(memcg); 1931 1932 if (locked) 1933 mem_cgroup_oom_notify(memcg); 1934 1935 mem_cgroup_unmark_under_oom(memcg); 1936 ret = mem_cgroup_out_of_memory(memcg, mask, order); 1937 1938 if (locked) 1939 mem_cgroup_oom_unlock(memcg); 1940 1941 return ret; 1942 } 1943 1944 /** 1945 * mem_cgroup_oom_synchronize - complete memcg OOM handling 1946 * @handle: actually kill/wait or just clean up the OOM state 1947 * 1948 * This has to be called at the end of a page fault if the memcg OOM 1949 * handler was enabled. 1950 * 1951 * Memcg supports userspace OOM handling where failed allocations must 1952 * sleep on a waitqueue until the userspace task resolves the 1953 * situation. Sleeping directly in the charge context with all kinds 1954 * of locks held is not a good idea, instead we remember an OOM state 1955 * in the task and mem_cgroup_oom_synchronize() has to be called at 1956 * the end of the page fault to complete the OOM handling. 1957 * 1958 * Returns %true if an ongoing memcg OOM situation was detected and 1959 * completed, %false otherwise. 1960 */ 1961 bool mem_cgroup_oom_synchronize(bool handle) 1962 { 1963 struct mem_cgroup *memcg = current->memcg_in_oom; 1964 struct oom_wait_info owait; 1965 bool locked; 1966 1967 /* OOM is global, do not handle */ 1968 if (!memcg) 1969 return false; 1970 1971 if (!handle) 1972 goto cleanup; 1973 1974 owait.memcg = memcg; 1975 owait.wait.flags = 0; 1976 owait.wait.func = memcg_oom_wake_function; 1977 owait.wait.private = current; 1978 INIT_LIST_HEAD(&owait.wait.entry); 1979 1980 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE); 1981 mem_cgroup_mark_under_oom(memcg); 1982 1983 locked = mem_cgroup_oom_trylock(memcg); 1984 1985 if (locked) 1986 mem_cgroup_oom_notify(memcg); 1987 1988 if (locked && !memcg->oom_kill_disable) { 1989 mem_cgroup_unmark_under_oom(memcg); 1990 finish_wait(&memcg_oom_waitq, &owait.wait); 1991 mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask, 1992 current->memcg_oom_order); 1993 } else { 1994 schedule(); 1995 mem_cgroup_unmark_under_oom(memcg); 1996 finish_wait(&memcg_oom_waitq, &owait.wait); 1997 } 1998 1999 if (locked) { 2000 mem_cgroup_oom_unlock(memcg); 2001 /* 2002 * There is no guarantee that an OOM-lock contender 2003 * sees the wakeups triggered by the OOM kill 2004 * uncharges. Wake any sleepers explicitly. 2005 */ 2006 memcg_oom_recover(memcg); 2007 } 2008 cleanup: 2009 current->memcg_in_oom = NULL; 2010 css_put(&memcg->css); 2011 return true; 2012 } 2013 2014 /** 2015 * mem_cgroup_get_oom_group - get a memory cgroup to clean up after OOM 2016 * @victim: task to be killed by the OOM killer 2017 * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM 2018 * 2019 * Returns a pointer to a memory cgroup, which has to be cleaned up 2020 * by killing all belonging OOM-killable tasks. 2021 * 2022 * Caller has to call mem_cgroup_put() on the returned non-NULL memcg. 2023 */ 2024 struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim, 2025 struct mem_cgroup *oom_domain) 2026 { 2027 struct mem_cgroup *oom_group = NULL; 2028 struct mem_cgroup *memcg; 2029 2030 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) 2031 return NULL; 2032 2033 if (!oom_domain) 2034 oom_domain = root_mem_cgroup; 2035 2036 rcu_read_lock(); 2037 2038 memcg = mem_cgroup_from_task(victim); 2039 if (memcg == root_mem_cgroup) 2040 goto out; 2041 2042 /* 2043 * If the victim task has been asynchronously moved to a different 2044 * memory cgroup, we might end up killing tasks outside oom_domain. 2045 * In this case it's better to ignore memory.group.oom. 2046 */ 2047 if (unlikely(!mem_cgroup_is_descendant(memcg, oom_domain))) 2048 goto out; 2049 2050 /* 2051 * Traverse the memory cgroup hierarchy from the victim task's 2052 * cgroup up to the OOMing cgroup (or root) to find the 2053 * highest-level memory cgroup with oom.group set. 2054 */ 2055 for (; memcg; memcg = parent_mem_cgroup(memcg)) { 2056 if (memcg->oom_group) 2057 oom_group = memcg; 2058 2059 if (memcg == oom_domain) 2060 break; 2061 } 2062 2063 if (oom_group) 2064 css_get(&oom_group->css); 2065 out: 2066 rcu_read_unlock(); 2067 2068 return oom_group; 2069 } 2070 2071 void mem_cgroup_print_oom_group(struct mem_cgroup *memcg) 2072 { 2073 pr_info("Tasks in "); 2074 pr_cont_cgroup_path(memcg->css.cgroup); 2075 pr_cont(" are going to be killed due to memory.oom.group set\n"); 2076 } 2077 2078 /** 2079 * folio_memcg_lock - Bind a folio to its memcg. 2080 * @folio: The folio. 2081 * 2082 * This function prevents unlocked LRU folios from being moved to 2083 * another cgroup. 2084 * 2085 * It ensures lifetime of the bound memcg. The caller is responsible 2086 * for the lifetime of the folio. 2087 */ 2088 void folio_memcg_lock(struct folio *folio) 2089 { 2090 struct mem_cgroup *memcg; 2091 unsigned long flags; 2092 2093 /* 2094 * The RCU lock is held throughout the transaction. The fast 2095 * path can get away without acquiring the memcg->move_lock 2096 * because page moving starts with an RCU grace period. 2097 */ 2098 rcu_read_lock(); 2099 2100 if (mem_cgroup_disabled()) 2101 return; 2102 again: 2103 memcg = folio_memcg(folio); 2104 if (unlikely(!memcg)) 2105 return; 2106 2107 #ifdef CONFIG_PROVE_LOCKING 2108 local_irq_save(flags); 2109 might_lock(&memcg->move_lock); 2110 local_irq_restore(flags); 2111 #endif 2112 2113 if (atomic_read(&memcg->moving_account) <= 0) 2114 return; 2115 2116 spin_lock_irqsave(&memcg->move_lock, flags); 2117 if (memcg != folio_memcg(folio)) { 2118 spin_unlock_irqrestore(&memcg->move_lock, flags); 2119 goto again; 2120 } 2121 2122 /* 2123 * When charge migration first begins, we can have multiple 2124 * critical sections holding the fast-path RCU lock and one 2125 * holding the slowpath move_lock. Track the task who has the 2126 * move_lock for unlock_page_memcg(). 2127 */ 2128 memcg->move_lock_task = current; 2129 memcg->move_lock_flags = flags; 2130 } 2131 2132 void lock_page_memcg(struct page *page) 2133 { 2134 folio_memcg_lock(page_folio(page)); 2135 } 2136 2137 static void __folio_memcg_unlock(struct mem_cgroup *memcg) 2138 { 2139 if (memcg && memcg->move_lock_task == current) { 2140 unsigned long flags = memcg->move_lock_flags; 2141 2142 memcg->move_lock_task = NULL; 2143 memcg->move_lock_flags = 0; 2144 2145 spin_unlock_irqrestore(&memcg->move_lock, flags); 2146 } 2147 2148 rcu_read_unlock(); 2149 } 2150 2151 /** 2152 * folio_memcg_unlock - Release the binding between a folio and its memcg. 2153 * @folio: The folio. 2154 * 2155 * This releases the binding created by folio_memcg_lock(). This does 2156 * not change the accounting of this folio to its memcg, but it does 2157 * permit others to change it. 2158 */ 2159 void folio_memcg_unlock(struct folio *folio) 2160 { 2161 __folio_memcg_unlock(folio_memcg(folio)); 2162 } 2163 2164 void unlock_page_memcg(struct page *page) 2165 { 2166 folio_memcg_unlock(page_folio(page)); 2167 } 2168 2169 struct memcg_stock_pcp { 2170 local_lock_t stock_lock; 2171 struct mem_cgroup *cached; /* this never be root cgroup */ 2172 unsigned int nr_pages; 2173 2174 #ifdef CONFIG_MEMCG_KMEM 2175 struct obj_cgroup *cached_objcg; 2176 struct pglist_data *cached_pgdat; 2177 unsigned int nr_bytes; 2178 int nr_slab_reclaimable_b; 2179 int nr_slab_unreclaimable_b; 2180 #endif 2181 2182 struct work_struct work; 2183 unsigned long flags; 2184 #define FLUSHING_CACHED_CHARGE 0 2185 }; 2186 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock) = { 2187 .stock_lock = INIT_LOCAL_LOCK(stock_lock), 2188 }; 2189 static DEFINE_MUTEX(percpu_charge_mutex); 2190 2191 #ifdef CONFIG_MEMCG_KMEM 2192 static struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock); 2193 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock, 2194 struct mem_cgroup *root_memcg); 2195 static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages); 2196 2197 #else 2198 static inline struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock) 2199 { 2200 return NULL; 2201 } 2202 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock, 2203 struct mem_cgroup *root_memcg) 2204 { 2205 return false; 2206 } 2207 static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages) 2208 { 2209 } 2210 #endif 2211 2212 /** 2213 * consume_stock: Try to consume stocked charge on this cpu. 2214 * @memcg: memcg to consume from. 2215 * @nr_pages: how many pages to charge. 2216 * 2217 * The charges will only happen if @memcg matches the current cpu's memcg 2218 * stock, and at least @nr_pages are available in that stock. Failure to 2219 * service an allocation will refill the stock. 2220 * 2221 * returns true if successful, false otherwise. 2222 */ 2223 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) 2224 { 2225 struct memcg_stock_pcp *stock; 2226 unsigned long flags; 2227 bool ret = false; 2228 2229 if (nr_pages > MEMCG_CHARGE_BATCH) 2230 return ret; 2231 2232 local_lock_irqsave(&memcg_stock.stock_lock, flags); 2233 2234 stock = this_cpu_ptr(&memcg_stock); 2235 if (memcg == stock->cached && stock->nr_pages >= nr_pages) { 2236 stock->nr_pages -= nr_pages; 2237 ret = true; 2238 } 2239 2240 local_unlock_irqrestore(&memcg_stock.stock_lock, flags); 2241 2242 return ret; 2243 } 2244 2245 /* 2246 * Returns stocks cached in percpu and reset cached information. 2247 */ 2248 static void drain_stock(struct memcg_stock_pcp *stock) 2249 { 2250 struct mem_cgroup *old = stock->cached; 2251 2252 if (!old) 2253 return; 2254 2255 if (stock->nr_pages) { 2256 page_counter_uncharge(&old->memory, stock->nr_pages); 2257 if (do_memsw_account()) 2258 page_counter_uncharge(&old->memsw, stock->nr_pages); 2259 stock->nr_pages = 0; 2260 } 2261 2262 css_put(&old->css); 2263 stock->cached = NULL; 2264 } 2265 2266 static void drain_local_stock(struct work_struct *dummy) 2267 { 2268 struct memcg_stock_pcp *stock; 2269 struct obj_cgroup *old = NULL; 2270 unsigned long flags; 2271 2272 /* 2273 * The only protection from cpu hotplug (memcg_hotplug_cpu_dead) vs. 2274 * drain_stock races is that we always operate on local CPU stock 2275 * here with IRQ disabled 2276 */ 2277 local_lock_irqsave(&memcg_stock.stock_lock, flags); 2278 2279 stock = this_cpu_ptr(&memcg_stock); 2280 old = drain_obj_stock(stock); 2281 drain_stock(stock); 2282 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags); 2283 2284 local_unlock_irqrestore(&memcg_stock.stock_lock, flags); 2285 if (old) 2286 obj_cgroup_put(old); 2287 } 2288 2289 /* 2290 * Cache charges(val) to local per_cpu area. 2291 * This will be consumed by consume_stock() function, later. 2292 */ 2293 static void __refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) 2294 { 2295 struct memcg_stock_pcp *stock; 2296 2297 stock = this_cpu_ptr(&memcg_stock); 2298 if (stock->cached != memcg) { /* reset if necessary */ 2299 drain_stock(stock); 2300 css_get(&memcg->css); 2301 stock->cached = memcg; 2302 } 2303 stock->nr_pages += nr_pages; 2304 2305 if (stock->nr_pages > MEMCG_CHARGE_BATCH) 2306 drain_stock(stock); 2307 } 2308 2309 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) 2310 { 2311 unsigned long flags; 2312 2313 local_lock_irqsave(&memcg_stock.stock_lock, flags); 2314 __refill_stock(memcg, nr_pages); 2315 local_unlock_irqrestore(&memcg_stock.stock_lock, flags); 2316 } 2317 2318 /* 2319 * Drains all per-CPU charge caches for given root_memcg resp. subtree 2320 * of the hierarchy under it. 2321 */ 2322 static void drain_all_stock(struct mem_cgroup *root_memcg) 2323 { 2324 int cpu, curcpu; 2325 2326 /* If someone's already draining, avoid adding running more workers. */ 2327 if (!mutex_trylock(&percpu_charge_mutex)) 2328 return; 2329 /* 2330 * Notify other cpus that system-wide "drain" is running 2331 * We do not care about races with the cpu hotplug because cpu down 2332 * as well as workers from this path always operate on the local 2333 * per-cpu data. CPU up doesn't touch memcg_stock at all. 2334 */ 2335 migrate_disable(); 2336 curcpu = smp_processor_id(); 2337 for_each_online_cpu(cpu) { 2338 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); 2339 struct mem_cgroup *memcg; 2340 bool flush = false; 2341 2342 rcu_read_lock(); 2343 memcg = stock->cached; 2344 if (memcg && stock->nr_pages && 2345 mem_cgroup_is_descendant(memcg, root_memcg)) 2346 flush = true; 2347 else if (obj_stock_flush_required(stock, root_memcg)) 2348 flush = true; 2349 rcu_read_unlock(); 2350 2351 if (flush && 2352 !test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) { 2353 if (cpu == curcpu) 2354 drain_local_stock(&stock->work); 2355 else 2356 schedule_work_on(cpu, &stock->work); 2357 } 2358 } 2359 migrate_enable(); 2360 mutex_unlock(&percpu_charge_mutex); 2361 } 2362 2363 static int memcg_hotplug_cpu_dead(unsigned int cpu) 2364 { 2365 struct memcg_stock_pcp *stock; 2366 2367 stock = &per_cpu(memcg_stock, cpu); 2368 drain_stock(stock); 2369 2370 return 0; 2371 } 2372 2373 static unsigned long reclaim_high(struct mem_cgroup *memcg, 2374 unsigned int nr_pages, 2375 gfp_t gfp_mask) 2376 { 2377 unsigned long nr_reclaimed = 0; 2378 2379 do { 2380 unsigned long pflags; 2381 2382 if (page_counter_read(&memcg->memory) <= 2383 READ_ONCE(memcg->memory.high)) 2384 continue; 2385 2386 memcg_memory_event(memcg, MEMCG_HIGH); 2387 2388 psi_memstall_enter(&pflags); 2389 nr_reclaimed += try_to_free_mem_cgroup_pages(memcg, nr_pages, 2390 gfp_mask, 2391 MEMCG_RECLAIM_MAY_SWAP); 2392 psi_memstall_leave(&pflags); 2393 } while ((memcg = parent_mem_cgroup(memcg)) && 2394 !mem_cgroup_is_root(memcg)); 2395 2396 return nr_reclaimed; 2397 } 2398 2399 static void high_work_func(struct work_struct *work) 2400 { 2401 struct mem_cgroup *memcg; 2402 2403 memcg = container_of(work, struct mem_cgroup, high_work); 2404 reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL); 2405 } 2406 2407 /* 2408 * Clamp the maximum sleep time per allocation batch to 2 seconds. This is 2409 * enough to still cause a significant slowdown in most cases, while still 2410 * allowing diagnostics and tracing to proceed without becoming stuck. 2411 */ 2412 #define MEMCG_MAX_HIGH_DELAY_JIFFIES (2UL*HZ) 2413 2414 /* 2415 * When calculating the delay, we use these either side of the exponentiation to 2416 * maintain precision and scale to a reasonable number of jiffies (see the table 2417 * below. 2418 * 2419 * - MEMCG_DELAY_PRECISION_SHIFT: Extra precision bits while translating the 2420 * overage ratio to a delay. 2421 * - MEMCG_DELAY_SCALING_SHIFT: The number of bits to scale down the 2422 * proposed penalty in order to reduce to a reasonable number of jiffies, and 2423 * to produce a reasonable delay curve. 2424 * 2425 * MEMCG_DELAY_SCALING_SHIFT just happens to be a number that produces a 2426 * reasonable delay curve compared to precision-adjusted overage, not 2427 * penalising heavily at first, but still making sure that growth beyond the 2428 * limit penalises misbehaviour cgroups by slowing them down exponentially. For 2429 * example, with a high of 100 megabytes: 2430 * 2431 * +-------+------------------------+ 2432 * | usage | time to allocate in ms | 2433 * +-------+------------------------+ 2434 * | 100M | 0 | 2435 * | 101M | 6 | 2436 * | 102M | 25 | 2437 * | 103M | 57 | 2438 * | 104M | 102 | 2439 * | 105M | 159 | 2440 * | 106M | 230 | 2441 * | 107M | 313 | 2442 * | 108M | 409 | 2443 * | 109M | 518 | 2444 * | 110M | 639 | 2445 * | 111M | 774 | 2446 * | 112M | 921 | 2447 * | 113M | 1081 | 2448 * | 114M | 1254 | 2449 * | 115M | 1439 | 2450 * | 116M | 1638 | 2451 * | 117M | 1849 | 2452 * | 118M | 2000 | 2453 * | 119M | 2000 | 2454 * | 120M | 2000 | 2455 * +-------+------------------------+ 2456 */ 2457 #define MEMCG_DELAY_PRECISION_SHIFT 20 2458 #define MEMCG_DELAY_SCALING_SHIFT 14 2459 2460 static u64 calculate_overage(unsigned long usage, unsigned long high) 2461 { 2462 u64 overage; 2463 2464 if (usage <= high) 2465 return 0; 2466 2467 /* 2468 * Prevent division by 0 in overage calculation by acting as if 2469 * it was a threshold of 1 page 2470 */ 2471 high = max(high, 1UL); 2472 2473 overage = usage - high; 2474 overage <<= MEMCG_DELAY_PRECISION_SHIFT; 2475 return div64_u64(overage, high); 2476 } 2477 2478 static u64 mem_find_max_overage(struct mem_cgroup *memcg) 2479 { 2480 u64 overage, max_overage = 0; 2481 2482 do { 2483 overage = calculate_overage(page_counter_read(&memcg->memory), 2484 READ_ONCE(memcg->memory.high)); 2485 max_overage = max(overage, max_overage); 2486 } while ((memcg = parent_mem_cgroup(memcg)) && 2487 !mem_cgroup_is_root(memcg)); 2488 2489 return max_overage; 2490 } 2491 2492 static u64 swap_find_max_overage(struct mem_cgroup *memcg) 2493 { 2494 u64 overage, max_overage = 0; 2495 2496 do { 2497 overage = calculate_overage(page_counter_read(&memcg->swap), 2498 READ_ONCE(memcg->swap.high)); 2499 if (overage) 2500 memcg_memory_event(memcg, MEMCG_SWAP_HIGH); 2501 max_overage = max(overage, max_overage); 2502 } while ((memcg = parent_mem_cgroup(memcg)) && 2503 !mem_cgroup_is_root(memcg)); 2504 2505 return max_overage; 2506 } 2507 2508 /* 2509 * Get the number of jiffies that we should penalise a mischievous cgroup which 2510 * is exceeding its memory.high by checking both it and its ancestors. 2511 */ 2512 static unsigned long calculate_high_delay(struct mem_cgroup *memcg, 2513 unsigned int nr_pages, 2514 u64 max_overage) 2515 { 2516 unsigned long penalty_jiffies; 2517 2518 if (!max_overage) 2519 return 0; 2520 2521 /* 2522 * We use overage compared to memory.high to calculate the number of 2523 * jiffies to sleep (penalty_jiffies). Ideally this value should be 2524 * fairly lenient on small overages, and increasingly harsh when the 2525 * memcg in question makes it clear that it has no intention of stopping 2526 * its crazy behaviour, so we exponentially increase the delay based on 2527 * overage amount. 2528 */ 2529 penalty_jiffies = max_overage * max_overage * HZ; 2530 penalty_jiffies >>= MEMCG_DELAY_PRECISION_SHIFT; 2531 penalty_jiffies >>= MEMCG_DELAY_SCALING_SHIFT; 2532 2533 /* 2534 * Factor in the task's own contribution to the overage, such that four 2535 * N-sized allocations are throttled approximately the same as one 2536 * 4N-sized allocation. 2537 * 2538 * MEMCG_CHARGE_BATCH pages is nominal, so work out how much smaller or 2539 * larger the current charge patch is than that. 2540 */ 2541 return penalty_jiffies * nr_pages / MEMCG_CHARGE_BATCH; 2542 } 2543 2544 /* 2545 * Scheduled by try_charge() to be executed from the userland return path 2546 * and reclaims memory over the high limit. 2547 */ 2548 void mem_cgroup_handle_over_high(void) 2549 { 2550 unsigned long penalty_jiffies; 2551 unsigned long pflags; 2552 unsigned long nr_reclaimed; 2553 unsigned int nr_pages = current->memcg_nr_pages_over_high; 2554 int nr_retries = MAX_RECLAIM_RETRIES; 2555 struct mem_cgroup *memcg; 2556 bool in_retry = false; 2557 2558 if (likely(!nr_pages)) 2559 return; 2560 2561 memcg = get_mem_cgroup_from_mm(current->mm); 2562 current->memcg_nr_pages_over_high = 0; 2563 2564 retry_reclaim: 2565 /* 2566 * The allocating task should reclaim at least the batch size, but for 2567 * subsequent retries we only want to do what's necessary to prevent oom 2568 * or breaching resource isolation. 2569 * 2570 * This is distinct from memory.max or page allocator behaviour because 2571 * memory.high is currently batched, whereas memory.max and the page 2572 * allocator run every time an allocation is made. 2573 */ 2574 nr_reclaimed = reclaim_high(memcg, 2575 in_retry ? SWAP_CLUSTER_MAX : nr_pages, 2576 GFP_KERNEL); 2577 2578 /* 2579 * memory.high is breached and reclaim is unable to keep up. Throttle 2580 * allocators proactively to slow down excessive growth. 2581 */ 2582 penalty_jiffies = calculate_high_delay(memcg, nr_pages, 2583 mem_find_max_overage(memcg)); 2584 2585 penalty_jiffies += calculate_high_delay(memcg, nr_pages, 2586 swap_find_max_overage(memcg)); 2587 2588 /* 2589 * Clamp the max delay per usermode return so as to still keep the 2590 * application moving forwards and also permit diagnostics, albeit 2591 * extremely slowly. 2592 */ 2593 penalty_jiffies = min(penalty_jiffies, MEMCG_MAX_HIGH_DELAY_JIFFIES); 2594 2595 /* 2596 * Don't sleep if the amount of jiffies this memcg owes us is so low 2597 * that it's not even worth doing, in an attempt to be nice to those who 2598 * go only a small amount over their memory.high value and maybe haven't 2599 * been aggressively reclaimed enough yet. 2600 */ 2601 if (penalty_jiffies <= HZ / 100) 2602 goto out; 2603 2604 /* 2605 * If reclaim is making forward progress but we're still over 2606 * memory.high, we want to encourage that rather than doing allocator 2607 * throttling. 2608 */ 2609 if (nr_reclaimed || nr_retries--) { 2610 in_retry = true; 2611 goto retry_reclaim; 2612 } 2613 2614 /* 2615 * If we exit early, we're guaranteed to die (since 2616 * schedule_timeout_killable sets TASK_KILLABLE). This means we don't 2617 * need to account for any ill-begotten jiffies to pay them off later. 2618 */ 2619 psi_memstall_enter(&pflags); 2620 schedule_timeout_killable(penalty_jiffies); 2621 psi_memstall_leave(&pflags); 2622 2623 out: 2624 css_put(&memcg->css); 2625 } 2626 2627 static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, 2628 unsigned int nr_pages) 2629 { 2630 unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages); 2631 int nr_retries = MAX_RECLAIM_RETRIES; 2632 struct mem_cgroup *mem_over_limit; 2633 struct page_counter *counter; 2634 unsigned long nr_reclaimed; 2635 bool passed_oom = false; 2636 unsigned int reclaim_options = MEMCG_RECLAIM_MAY_SWAP; 2637 bool drained = false; 2638 bool raised_max_event = false; 2639 unsigned long pflags; 2640 2641 retry: 2642 if (consume_stock(memcg, nr_pages)) 2643 return 0; 2644 2645 if (!do_memsw_account() || 2646 page_counter_try_charge(&memcg->memsw, batch, &counter)) { 2647 if (page_counter_try_charge(&memcg->memory, batch, &counter)) 2648 goto done_restock; 2649 if (do_memsw_account()) 2650 page_counter_uncharge(&memcg->memsw, batch); 2651 mem_over_limit = mem_cgroup_from_counter(counter, memory); 2652 } else { 2653 mem_over_limit = mem_cgroup_from_counter(counter, memsw); 2654 reclaim_options &= ~MEMCG_RECLAIM_MAY_SWAP; 2655 } 2656 2657 if (batch > nr_pages) { 2658 batch = nr_pages; 2659 goto retry; 2660 } 2661 2662 /* 2663 * Prevent unbounded recursion when reclaim operations need to 2664 * allocate memory. This might exceed the limits temporarily, 2665 * but we prefer facilitating memory reclaim and getting back 2666 * under the limit over triggering OOM kills in these cases. 2667 */ 2668 if (unlikely(current->flags & PF_MEMALLOC)) 2669 goto force; 2670 2671 if (unlikely(task_in_memcg_oom(current))) 2672 goto nomem; 2673 2674 if (!gfpflags_allow_blocking(gfp_mask)) 2675 goto nomem; 2676 2677 memcg_memory_event(mem_over_limit, MEMCG_MAX); 2678 raised_max_event = true; 2679 2680 psi_memstall_enter(&pflags); 2681 nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages, 2682 gfp_mask, reclaim_options); 2683 psi_memstall_leave(&pflags); 2684 2685 if (mem_cgroup_margin(mem_over_limit) >= nr_pages) 2686 goto retry; 2687 2688 if (!drained) { 2689 drain_all_stock(mem_over_limit); 2690 drained = true; 2691 goto retry; 2692 } 2693 2694 if (gfp_mask & __GFP_NORETRY) 2695 goto nomem; 2696 /* 2697 * Even though the limit is exceeded at this point, reclaim 2698 * may have been able to free some pages. Retry the charge 2699 * before killing the task. 2700 * 2701 * Only for regular pages, though: huge pages are rather 2702 * unlikely to succeed so close to the limit, and we fall back 2703 * to regular pages anyway in case of failure. 2704 */ 2705 if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER)) 2706 goto retry; 2707 /* 2708 * At task move, charge accounts can be doubly counted. So, it's 2709 * better to wait until the end of task_move if something is going on. 2710 */ 2711 if (mem_cgroup_wait_acct_move(mem_over_limit)) 2712 goto retry; 2713 2714 if (nr_retries--) 2715 goto retry; 2716 2717 if (gfp_mask & __GFP_RETRY_MAYFAIL) 2718 goto nomem; 2719 2720 /* Avoid endless loop for tasks bypassed by the oom killer */ 2721 if (passed_oom && task_is_dying()) 2722 goto nomem; 2723 2724 /* 2725 * keep retrying as long as the memcg oom killer is able to make 2726 * a forward progress or bypass the charge if the oom killer 2727 * couldn't make any progress. 2728 */ 2729 if (mem_cgroup_oom(mem_over_limit, gfp_mask, 2730 get_order(nr_pages * PAGE_SIZE))) { 2731 passed_oom = true; 2732 nr_retries = MAX_RECLAIM_RETRIES; 2733 goto retry; 2734 } 2735 nomem: 2736 /* 2737 * Memcg doesn't have a dedicated reserve for atomic 2738 * allocations. But like the global atomic pool, we need to 2739 * put the burden of reclaim on regular allocation requests 2740 * and let these go through as privileged allocations. 2741 */ 2742 if (!(gfp_mask & (__GFP_NOFAIL | __GFP_HIGH))) 2743 return -ENOMEM; 2744 force: 2745 /* 2746 * If the allocation has to be enforced, don't forget to raise 2747 * a MEMCG_MAX event. 2748 */ 2749 if (!raised_max_event) 2750 memcg_memory_event(mem_over_limit, MEMCG_MAX); 2751 2752 /* 2753 * The allocation either can't fail or will lead to more memory 2754 * being freed very soon. Allow memory usage go over the limit 2755 * temporarily by force charging it. 2756 */ 2757 page_counter_charge(&memcg->memory, nr_pages); 2758 if (do_memsw_account()) 2759 page_counter_charge(&memcg->memsw, nr_pages); 2760 2761 return 0; 2762 2763 done_restock: 2764 if (batch > nr_pages) 2765 refill_stock(memcg, batch - nr_pages); 2766 2767 /* 2768 * If the hierarchy is above the normal consumption range, schedule 2769 * reclaim on returning to userland. We can perform reclaim here 2770 * if __GFP_RECLAIM but let's always punt for simplicity and so that 2771 * GFP_KERNEL can consistently be used during reclaim. @memcg is 2772 * not recorded as it most likely matches current's and won't 2773 * change in the meantime. As high limit is checked again before 2774 * reclaim, the cost of mismatch is negligible. 2775 */ 2776 do { 2777 bool mem_high, swap_high; 2778 2779 mem_high = page_counter_read(&memcg->memory) > 2780 READ_ONCE(memcg->memory.high); 2781 swap_high = page_counter_read(&memcg->swap) > 2782 READ_ONCE(memcg->swap.high); 2783 2784 /* Don't bother a random interrupted task */ 2785 if (!in_task()) { 2786 if (mem_high) { 2787 schedule_work(&memcg->high_work); 2788 break; 2789 } 2790 continue; 2791 } 2792 2793 if (mem_high || swap_high) { 2794 /* 2795 * The allocating tasks in this cgroup will need to do 2796 * reclaim or be throttled to prevent further growth 2797 * of the memory or swap footprints. 2798 * 2799 * Target some best-effort fairness between the tasks, 2800 * and distribute reclaim work and delay penalties 2801 * based on how much each task is actually allocating. 2802 */ 2803 current->memcg_nr_pages_over_high += batch; 2804 set_notify_resume(current); 2805 break; 2806 } 2807 } while ((memcg = parent_mem_cgroup(memcg))); 2808 2809 if (current->memcg_nr_pages_over_high > MEMCG_CHARGE_BATCH && 2810 !(current->flags & PF_MEMALLOC) && 2811 gfpflags_allow_blocking(gfp_mask)) { 2812 mem_cgroup_handle_over_high(); 2813 } 2814 return 0; 2815 } 2816 2817 static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, 2818 unsigned int nr_pages) 2819 { 2820 if (mem_cgroup_is_root(memcg)) 2821 return 0; 2822 2823 return try_charge_memcg(memcg, gfp_mask, nr_pages); 2824 } 2825 2826 static inline void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages) 2827 { 2828 if (mem_cgroup_is_root(memcg)) 2829 return; 2830 2831 page_counter_uncharge(&memcg->memory, nr_pages); 2832 if (do_memsw_account()) 2833 page_counter_uncharge(&memcg->memsw, nr_pages); 2834 } 2835 2836 static void commit_charge(struct folio *folio, struct mem_cgroup *memcg) 2837 { 2838 VM_BUG_ON_FOLIO(folio_memcg(folio), folio); 2839 /* 2840 * Any of the following ensures page's memcg stability: 2841 * 2842 * - the page lock 2843 * - LRU isolation 2844 * - lock_page_memcg() 2845 * - exclusive reference 2846 * - mem_cgroup_trylock_pages() 2847 */ 2848 folio->memcg_data = (unsigned long)memcg; 2849 } 2850 2851 #ifdef CONFIG_MEMCG_KMEM 2852 /* 2853 * The allocated objcg pointers array is not accounted directly. 2854 * Moreover, it should not come from DMA buffer and is not readily 2855 * reclaimable. So those GFP bits should be masked off. 2856 */ 2857 #define OBJCGS_CLEAR_MASK (__GFP_DMA | __GFP_RECLAIMABLE | __GFP_ACCOUNT) 2858 2859 /* 2860 * mod_objcg_mlstate() may be called with irq enabled, so 2861 * mod_memcg_lruvec_state() should be used. 2862 */ 2863 static inline void mod_objcg_mlstate(struct obj_cgroup *objcg, 2864 struct pglist_data *pgdat, 2865 enum node_stat_item idx, int nr) 2866 { 2867 struct mem_cgroup *memcg; 2868 struct lruvec *lruvec; 2869 2870 rcu_read_lock(); 2871 memcg = obj_cgroup_memcg(objcg); 2872 lruvec = mem_cgroup_lruvec(memcg, pgdat); 2873 mod_memcg_lruvec_state(lruvec, idx, nr); 2874 rcu_read_unlock(); 2875 } 2876 2877 int memcg_alloc_slab_cgroups(struct slab *slab, struct kmem_cache *s, 2878 gfp_t gfp, bool new_slab) 2879 { 2880 unsigned int objects = objs_per_slab(s, slab); 2881 unsigned long memcg_data; 2882 void *vec; 2883 2884 gfp &= ~OBJCGS_CLEAR_MASK; 2885 vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp, 2886 slab_nid(slab)); 2887 if (!vec) 2888 return -ENOMEM; 2889 2890 memcg_data = (unsigned long) vec | MEMCG_DATA_OBJCGS; 2891 if (new_slab) { 2892 /* 2893 * If the slab is brand new and nobody can yet access its 2894 * memcg_data, no synchronization is required and memcg_data can 2895 * be simply assigned. 2896 */ 2897 slab->memcg_data = memcg_data; 2898 } else if (cmpxchg(&slab->memcg_data, 0, memcg_data)) { 2899 /* 2900 * If the slab is already in use, somebody can allocate and 2901 * assign obj_cgroups in parallel. In this case the existing 2902 * objcg vector should be reused. 2903 */ 2904 kfree(vec); 2905 return 0; 2906 } 2907 2908 kmemleak_not_leak(vec); 2909 return 0; 2910 } 2911 2912 static __always_inline 2913 struct mem_cgroup *mem_cgroup_from_obj_folio(struct folio *folio, void *p) 2914 { 2915 /* 2916 * Slab objects are accounted individually, not per-page. 2917 * Memcg membership data for each individual object is saved in 2918 * slab->memcg_data. 2919 */ 2920 if (folio_test_slab(folio)) { 2921 struct obj_cgroup **objcgs; 2922 struct slab *slab; 2923 unsigned int off; 2924 2925 slab = folio_slab(folio); 2926 objcgs = slab_objcgs(slab); 2927 if (!objcgs) 2928 return NULL; 2929 2930 off = obj_to_index(slab->slab_cache, slab, p); 2931 if (objcgs[off]) 2932 return obj_cgroup_memcg(objcgs[off]); 2933 2934 return NULL; 2935 } 2936 2937 /* 2938 * page_memcg_check() is used here, because in theory we can encounter 2939 * a folio where the slab flag has been cleared already, but 2940 * slab->memcg_data has not been freed yet 2941 * page_memcg_check(page) will guarantee that a proper memory 2942 * cgroup pointer or NULL will be returned. 2943 */ 2944 return page_memcg_check(folio_page(folio, 0)); 2945 } 2946 2947 /* 2948 * Returns a pointer to the memory cgroup to which the kernel object is charged. 2949 * 2950 * A passed kernel object can be a slab object, vmalloc object or a generic 2951 * kernel page, so different mechanisms for getting the memory cgroup pointer 2952 * should be used. 2953 * 2954 * In certain cases (e.g. kernel stacks or large kmallocs with SLUB) the caller 2955 * can not know for sure how the kernel object is implemented. 2956 * mem_cgroup_from_obj() can be safely used in such cases. 2957 * 2958 * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(), 2959 * cgroup_mutex, etc. 2960 */ 2961 struct mem_cgroup *mem_cgroup_from_obj(void *p) 2962 { 2963 struct folio *folio; 2964 2965 if (mem_cgroup_disabled()) 2966 return NULL; 2967 2968 if (unlikely(is_vmalloc_addr(p))) 2969 folio = page_folio(vmalloc_to_page(p)); 2970 else 2971 folio = virt_to_folio(p); 2972 2973 return mem_cgroup_from_obj_folio(folio, p); 2974 } 2975 2976 /* 2977 * Returns a pointer to the memory cgroup to which the kernel object is charged. 2978 * Similar to mem_cgroup_from_obj(), but faster and not suitable for objects, 2979 * allocated using vmalloc(). 2980 * 2981 * A passed kernel object must be a slab object or a generic kernel page. 2982 * 2983 * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(), 2984 * cgroup_mutex, etc. 2985 */ 2986 struct mem_cgroup *mem_cgroup_from_slab_obj(void *p) 2987 { 2988 if (mem_cgroup_disabled()) 2989 return NULL; 2990 2991 return mem_cgroup_from_obj_folio(virt_to_folio(p), p); 2992 } 2993 2994 static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg) 2995 { 2996 struct obj_cgroup *objcg = NULL; 2997 2998 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) { 2999 objcg = rcu_dereference(memcg->objcg); 3000 if (objcg && obj_cgroup_tryget(objcg)) 3001 break; 3002 objcg = NULL; 3003 } 3004 return objcg; 3005 } 3006 3007 __always_inline struct obj_cgroup *get_obj_cgroup_from_current(void) 3008 { 3009 struct obj_cgroup *objcg = NULL; 3010 struct mem_cgroup *memcg; 3011 3012 if (memcg_kmem_bypass()) 3013 return NULL; 3014 3015 rcu_read_lock(); 3016 if (unlikely(active_memcg())) 3017 memcg = active_memcg(); 3018 else 3019 memcg = mem_cgroup_from_task(current); 3020 objcg = __get_obj_cgroup_from_memcg(memcg); 3021 rcu_read_unlock(); 3022 return objcg; 3023 } 3024 3025 struct obj_cgroup *get_obj_cgroup_from_page(struct page *page) 3026 { 3027 struct obj_cgroup *objcg; 3028 3029 if (!memcg_kmem_enabled()) 3030 return NULL; 3031 3032 if (PageMemcgKmem(page)) { 3033 objcg = __folio_objcg(page_folio(page)); 3034 obj_cgroup_get(objcg); 3035 } else { 3036 struct mem_cgroup *memcg; 3037 3038 rcu_read_lock(); 3039 memcg = __folio_memcg(page_folio(page)); 3040 if (memcg) 3041 objcg = __get_obj_cgroup_from_memcg(memcg); 3042 else 3043 objcg = NULL; 3044 rcu_read_unlock(); 3045 } 3046 return objcg; 3047 } 3048 3049 static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages) 3050 { 3051 mod_memcg_state(memcg, MEMCG_KMEM, nr_pages); 3052 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) { 3053 if (nr_pages > 0) 3054 page_counter_charge(&memcg->kmem, nr_pages); 3055 else 3056 page_counter_uncharge(&memcg->kmem, -nr_pages); 3057 } 3058 } 3059 3060 3061 /* 3062 * obj_cgroup_uncharge_pages: uncharge a number of kernel pages from a objcg 3063 * @objcg: object cgroup to uncharge 3064 * @nr_pages: number of pages to uncharge 3065 */ 3066 static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg, 3067 unsigned int nr_pages) 3068 { 3069 struct mem_cgroup *memcg; 3070 3071 memcg = get_mem_cgroup_from_objcg(objcg); 3072 3073 memcg_account_kmem(memcg, -nr_pages); 3074 refill_stock(memcg, nr_pages); 3075 3076 css_put(&memcg->css); 3077 } 3078 3079 /* 3080 * obj_cgroup_charge_pages: charge a number of kernel pages to a objcg 3081 * @objcg: object cgroup to charge 3082 * @gfp: reclaim mode 3083 * @nr_pages: number of pages to charge 3084 * 3085 * Returns 0 on success, an error code on failure. 3086 */ 3087 static int obj_cgroup_charge_pages(struct obj_cgroup *objcg, gfp_t gfp, 3088 unsigned int nr_pages) 3089 { 3090 struct mem_cgroup *memcg; 3091 int ret; 3092 3093 memcg = get_mem_cgroup_from_objcg(objcg); 3094 3095 ret = try_charge_memcg(memcg, gfp, nr_pages); 3096 if (ret) 3097 goto out; 3098 3099 memcg_account_kmem(memcg, nr_pages); 3100 out: 3101 css_put(&memcg->css); 3102 3103 return ret; 3104 } 3105 3106 /** 3107 * __memcg_kmem_charge_page: charge a kmem page to the current memory cgroup 3108 * @page: page to charge 3109 * @gfp: reclaim mode 3110 * @order: allocation order 3111 * 3112 * Returns 0 on success, an error code on failure. 3113 */ 3114 int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order) 3115 { 3116 struct obj_cgroup *objcg; 3117 int ret = 0; 3118 3119 objcg = get_obj_cgroup_from_current(); 3120 if (objcg) { 3121 ret = obj_cgroup_charge_pages(objcg, gfp, 1 << order); 3122 if (!ret) { 3123 page->memcg_data = (unsigned long)objcg | 3124 MEMCG_DATA_KMEM; 3125 return 0; 3126 } 3127 obj_cgroup_put(objcg); 3128 } 3129 return ret; 3130 } 3131 3132 /** 3133 * __memcg_kmem_uncharge_page: uncharge a kmem page 3134 * @page: page to uncharge 3135 * @order: allocation order 3136 */ 3137 void __memcg_kmem_uncharge_page(struct page *page, int order) 3138 { 3139 struct folio *folio = page_folio(page); 3140 struct obj_cgroup *objcg; 3141 unsigned int nr_pages = 1 << order; 3142 3143 if (!folio_memcg_kmem(folio)) 3144 return; 3145 3146 objcg = __folio_objcg(folio); 3147 obj_cgroup_uncharge_pages(objcg, nr_pages); 3148 folio->memcg_data = 0; 3149 obj_cgroup_put(objcg); 3150 } 3151 3152 void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat, 3153 enum node_stat_item idx, int nr) 3154 { 3155 struct memcg_stock_pcp *stock; 3156 struct obj_cgroup *old = NULL; 3157 unsigned long flags; 3158 int *bytes; 3159 3160 local_lock_irqsave(&memcg_stock.stock_lock, flags); 3161 stock = this_cpu_ptr(&memcg_stock); 3162 3163 /* 3164 * Save vmstat data in stock and skip vmstat array update unless 3165 * accumulating over a page of vmstat data or when pgdat or idx 3166 * changes. 3167 */ 3168 if (stock->cached_objcg != objcg) { 3169 old = drain_obj_stock(stock); 3170 obj_cgroup_get(objcg); 3171 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes) 3172 ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0; 3173 stock->cached_objcg = objcg; 3174 stock->cached_pgdat = pgdat; 3175 } else if (stock->cached_pgdat != pgdat) { 3176 /* Flush the existing cached vmstat data */ 3177 struct pglist_data *oldpg = stock->cached_pgdat; 3178 3179 if (stock->nr_slab_reclaimable_b) { 3180 mod_objcg_mlstate(objcg, oldpg, NR_SLAB_RECLAIMABLE_B, 3181 stock->nr_slab_reclaimable_b); 3182 stock->nr_slab_reclaimable_b = 0; 3183 } 3184 if (stock->nr_slab_unreclaimable_b) { 3185 mod_objcg_mlstate(objcg, oldpg, NR_SLAB_UNRECLAIMABLE_B, 3186 stock->nr_slab_unreclaimable_b); 3187 stock->nr_slab_unreclaimable_b = 0; 3188 } 3189 stock->cached_pgdat = pgdat; 3190 } 3191 3192 bytes = (idx == NR_SLAB_RECLAIMABLE_B) ? &stock->nr_slab_reclaimable_b 3193 : &stock->nr_slab_unreclaimable_b; 3194 /* 3195 * Even for large object >= PAGE_SIZE, the vmstat data will still be 3196 * cached locally at least once before pushing it out. 3197 */ 3198 if (!*bytes) { 3199 *bytes = nr; 3200 nr = 0; 3201 } else { 3202 *bytes += nr; 3203 if (abs(*bytes) > PAGE_SIZE) { 3204 nr = *bytes; 3205 *bytes = 0; 3206 } else { 3207 nr = 0; 3208 } 3209 } 3210 if (nr) 3211 mod_objcg_mlstate(objcg, pgdat, idx, nr); 3212 3213 local_unlock_irqrestore(&memcg_stock.stock_lock, flags); 3214 if (old) 3215 obj_cgroup_put(old); 3216 } 3217 3218 static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes) 3219 { 3220 struct memcg_stock_pcp *stock; 3221 unsigned long flags; 3222 bool ret = false; 3223 3224 local_lock_irqsave(&memcg_stock.stock_lock, flags); 3225 3226 stock = this_cpu_ptr(&memcg_stock); 3227 if (objcg == stock->cached_objcg && stock->nr_bytes >= nr_bytes) { 3228 stock->nr_bytes -= nr_bytes; 3229 ret = true; 3230 } 3231 3232 local_unlock_irqrestore(&memcg_stock.stock_lock, flags); 3233 3234 return ret; 3235 } 3236 3237 static struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock) 3238 { 3239 struct obj_cgroup *old = stock->cached_objcg; 3240 3241 if (!old) 3242 return NULL; 3243 3244 if (stock->nr_bytes) { 3245 unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT; 3246 unsigned int nr_bytes = stock->nr_bytes & (PAGE_SIZE - 1); 3247 3248 if (nr_pages) { 3249 struct mem_cgroup *memcg; 3250 3251 memcg = get_mem_cgroup_from_objcg(old); 3252 3253 memcg_account_kmem(memcg, -nr_pages); 3254 __refill_stock(memcg, nr_pages); 3255 3256 css_put(&memcg->css); 3257 } 3258 3259 /* 3260 * The leftover is flushed to the centralized per-memcg value. 3261 * On the next attempt to refill obj stock it will be moved 3262 * to a per-cpu stock (probably, on an other CPU), see 3263 * refill_obj_stock(). 3264 * 3265 * How often it's flushed is a trade-off between the memory 3266 * limit enforcement accuracy and potential CPU contention, 3267 * so it might be changed in the future. 3268 */ 3269 atomic_add(nr_bytes, &old->nr_charged_bytes); 3270 stock->nr_bytes = 0; 3271 } 3272 3273 /* 3274 * Flush the vmstat data in current stock 3275 */ 3276 if (stock->nr_slab_reclaimable_b || stock->nr_slab_unreclaimable_b) { 3277 if (stock->nr_slab_reclaimable_b) { 3278 mod_objcg_mlstate(old, stock->cached_pgdat, 3279 NR_SLAB_RECLAIMABLE_B, 3280 stock->nr_slab_reclaimable_b); 3281 stock->nr_slab_reclaimable_b = 0; 3282 } 3283 if (stock->nr_slab_unreclaimable_b) { 3284 mod_objcg_mlstate(old, stock->cached_pgdat, 3285 NR_SLAB_UNRECLAIMABLE_B, 3286 stock->nr_slab_unreclaimable_b); 3287 stock->nr_slab_unreclaimable_b = 0; 3288 } 3289 stock->cached_pgdat = NULL; 3290 } 3291 3292 stock->cached_objcg = NULL; 3293 /* 3294 * The `old' objects needs to be released by the caller via 3295 * obj_cgroup_put() outside of memcg_stock_pcp::stock_lock. 3296 */ 3297 return old; 3298 } 3299 3300 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock, 3301 struct mem_cgroup *root_memcg) 3302 { 3303 struct mem_cgroup *memcg; 3304 3305 if (stock->cached_objcg) { 3306 memcg = obj_cgroup_memcg(stock->cached_objcg); 3307 if (memcg && mem_cgroup_is_descendant(memcg, root_memcg)) 3308 return true; 3309 } 3310 3311 return false; 3312 } 3313 3314 static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes, 3315 bool allow_uncharge) 3316 { 3317 struct memcg_stock_pcp *stock; 3318 struct obj_cgroup *old = NULL; 3319 unsigned long flags; 3320 unsigned int nr_pages = 0; 3321 3322 local_lock_irqsave(&memcg_stock.stock_lock, flags); 3323 3324 stock = this_cpu_ptr(&memcg_stock); 3325 if (stock->cached_objcg != objcg) { /* reset if necessary */ 3326 old = drain_obj_stock(stock); 3327 obj_cgroup_get(objcg); 3328 stock->cached_objcg = objcg; 3329 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes) 3330 ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0; 3331 allow_uncharge = true; /* Allow uncharge when objcg changes */ 3332 } 3333 stock->nr_bytes += nr_bytes; 3334 3335 if (allow_uncharge && (stock->nr_bytes > PAGE_SIZE)) { 3336 nr_pages = stock->nr_bytes >> PAGE_SHIFT; 3337 stock->nr_bytes &= (PAGE_SIZE - 1); 3338 } 3339 3340 local_unlock_irqrestore(&memcg_stock.stock_lock, flags); 3341 if (old) 3342 obj_cgroup_put(old); 3343 3344 if (nr_pages) 3345 obj_cgroup_uncharge_pages(objcg, nr_pages); 3346 } 3347 3348 int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size) 3349 { 3350 unsigned int nr_pages, nr_bytes; 3351 int ret; 3352 3353 if (consume_obj_stock(objcg, size)) 3354 return 0; 3355 3356 /* 3357 * In theory, objcg->nr_charged_bytes can have enough 3358 * pre-charged bytes to satisfy the allocation. However, 3359 * flushing objcg->nr_charged_bytes requires two atomic 3360 * operations, and objcg->nr_charged_bytes can't be big. 3361 * The shared objcg->nr_charged_bytes can also become a 3362 * performance bottleneck if all tasks of the same memcg are 3363 * trying to update it. So it's better to ignore it and try 3364 * grab some new pages. The stock's nr_bytes will be flushed to 3365 * objcg->nr_charged_bytes later on when objcg changes. 3366 * 3367 * The stock's nr_bytes may contain enough pre-charged bytes 3368 * to allow one less page from being charged, but we can't rely 3369 * on the pre-charged bytes not being changed outside of 3370 * consume_obj_stock() or refill_obj_stock(). So ignore those 3371 * pre-charged bytes as well when charging pages. To avoid a 3372 * page uncharge right after a page charge, we set the 3373 * allow_uncharge flag to false when calling refill_obj_stock() 3374 * to temporarily allow the pre-charged bytes to exceed the page 3375 * size limit. The maximum reachable value of the pre-charged 3376 * bytes is (sizeof(object) + PAGE_SIZE - 2) if there is no data 3377 * race. 3378 */ 3379 nr_pages = size >> PAGE_SHIFT; 3380 nr_bytes = size & (PAGE_SIZE - 1); 3381 3382 if (nr_bytes) 3383 nr_pages += 1; 3384 3385 ret = obj_cgroup_charge_pages(objcg, gfp, nr_pages); 3386 if (!ret && nr_bytes) 3387 refill_obj_stock(objcg, PAGE_SIZE - nr_bytes, false); 3388 3389 return ret; 3390 } 3391 3392 void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size) 3393 { 3394 refill_obj_stock(objcg, size, true); 3395 } 3396 3397 #endif /* CONFIG_MEMCG_KMEM */ 3398 3399 /* 3400 * Because page_memcg(head) is not set on tails, set it now. 3401 */ 3402 void split_page_memcg(struct page *head, unsigned int nr) 3403 { 3404 struct folio *folio = page_folio(head); 3405 struct mem_cgroup *memcg = folio_memcg(folio); 3406 int i; 3407 3408 if (mem_cgroup_disabled() || !memcg) 3409 return; 3410 3411 for (i = 1; i < nr; i++) 3412 folio_page(folio, i)->memcg_data = folio->memcg_data; 3413 3414 if (folio_memcg_kmem(folio)) 3415 obj_cgroup_get_many(__folio_objcg(folio), nr - 1); 3416 else 3417 css_get_many(&memcg->css, nr - 1); 3418 } 3419 3420 #ifdef CONFIG_SWAP 3421 /** 3422 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record. 3423 * @entry: swap entry to be moved 3424 * @from: mem_cgroup which the entry is moved from 3425 * @to: mem_cgroup which the entry is moved to 3426 * 3427 * It succeeds only when the swap_cgroup's record for this entry is the same 3428 * as the mem_cgroup's id of @from. 3429 * 3430 * Returns 0 on success, -EINVAL on failure. 3431 * 3432 * The caller must have charged to @to, IOW, called page_counter_charge() about 3433 * both res and memsw, and called css_get(). 3434 */ 3435 static int mem_cgroup_move_swap_account(swp_entry_t entry, 3436 struct mem_cgroup *from, struct mem_cgroup *to) 3437 { 3438 unsigned short old_id, new_id; 3439 3440 old_id = mem_cgroup_id(from); 3441 new_id = mem_cgroup_id(to); 3442 3443 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) { 3444 mod_memcg_state(from, MEMCG_SWAP, -1); 3445 mod_memcg_state(to, MEMCG_SWAP, 1); 3446 return 0; 3447 } 3448 return -EINVAL; 3449 } 3450 #else 3451 static inline int mem_cgroup_move_swap_account(swp_entry_t entry, 3452 struct mem_cgroup *from, struct mem_cgroup *to) 3453 { 3454 return -EINVAL; 3455 } 3456 #endif 3457 3458 static DEFINE_MUTEX(memcg_max_mutex); 3459 3460 static int mem_cgroup_resize_max(struct mem_cgroup *memcg, 3461 unsigned long max, bool memsw) 3462 { 3463 bool enlarge = false; 3464 bool drained = false; 3465 int ret; 3466 bool limits_invariant; 3467 struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory; 3468 3469 do { 3470 if (signal_pending(current)) { 3471 ret = -EINTR; 3472 break; 3473 } 3474 3475 mutex_lock(&memcg_max_mutex); 3476 /* 3477 * Make sure that the new limit (memsw or memory limit) doesn't 3478 * break our basic invariant rule memory.max <= memsw.max. 3479 */ 3480 limits_invariant = memsw ? max >= READ_ONCE(memcg->memory.max) : 3481 max <= memcg->memsw.max; 3482 if (!limits_invariant) { 3483 mutex_unlock(&memcg_max_mutex); 3484 ret = -EINVAL; 3485 break; 3486 } 3487 if (max > counter->max) 3488 enlarge = true; 3489 ret = page_counter_set_max(counter, max); 3490 mutex_unlock(&memcg_max_mutex); 3491 3492 if (!ret) 3493 break; 3494 3495 if (!drained) { 3496 drain_all_stock(memcg); 3497 drained = true; 3498 continue; 3499 } 3500 3501 if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, 3502 memsw ? 0 : MEMCG_RECLAIM_MAY_SWAP)) { 3503 ret = -EBUSY; 3504 break; 3505 } 3506 } while (true); 3507 3508 if (!ret && enlarge) 3509 memcg_oom_recover(memcg); 3510 3511 return ret; 3512 } 3513 3514 unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order, 3515 gfp_t gfp_mask, 3516 unsigned long *total_scanned) 3517 { 3518 unsigned long nr_reclaimed = 0; 3519 struct mem_cgroup_per_node *mz, *next_mz = NULL; 3520 unsigned long reclaimed; 3521 int loop = 0; 3522 struct mem_cgroup_tree_per_node *mctz; 3523 unsigned long excess; 3524 3525 if (order > 0) 3526 return 0; 3527 3528 mctz = soft_limit_tree.rb_tree_per_node[pgdat->node_id]; 3529 3530 /* 3531 * Do not even bother to check the largest node if the root 3532 * is empty. Do it lockless to prevent lock bouncing. Races 3533 * are acceptable as soft limit is best effort anyway. 3534 */ 3535 if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root)) 3536 return 0; 3537 3538 /* 3539 * This loop can run a while, specially if mem_cgroup's continuously 3540 * keep exceeding their soft limit and putting the system under 3541 * pressure 3542 */ 3543 do { 3544 if (next_mz) 3545 mz = next_mz; 3546 else 3547 mz = mem_cgroup_largest_soft_limit_node(mctz); 3548 if (!mz) 3549 break; 3550 3551 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat, 3552 gfp_mask, total_scanned); 3553 nr_reclaimed += reclaimed; 3554 spin_lock_irq(&mctz->lock); 3555 3556 /* 3557 * If we failed to reclaim anything from this memory cgroup 3558 * it is time to move on to the next cgroup 3559 */ 3560 next_mz = NULL; 3561 if (!reclaimed) 3562 next_mz = __mem_cgroup_largest_soft_limit_node(mctz); 3563 3564 excess = soft_limit_excess(mz->memcg); 3565 /* 3566 * One school of thought says that we should not add 3567 * back the node to the tree if reclaim returns 0. 3568 * But our reclaim could return 0, simply because due 3569 * to priority we are exposing a smaller subset of 3570 * memory to reclaim from. Consider this as a longer 3571 * term TODO. 3572 */ 3573 /* If excess == 0, no tree ops */ 3574 __mem_cgroup_insert_exceeded(mz, mctz, excess); 3575 spin_unlock_irq(&mctz->lock); 3576 css_put(&mz->memcg->css); 3577 loop++; 3578 /* 3579 * Could not reclaim anything and there are no more 3580 * mem cgroups to try or we seem to be looping without 3581 * reclaiming anything. 3582 */ 3583 if (!nr_reclaimed && 3584 (next_mz == NULL || 3585 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS)) 3586 break; 3587 } while (!nr_reclaimed); 3588 if (next_mz) 3589 css_put(&next_mz->memcg->css); 3590 return nr_reclaimed; 3591 } 3592 3593 /* 3594 * Reclaims as many pages from the given memcg as possible. 3595 * 3596 * Caller is responsible for holding css reference for memcg. 3597 */ 3598 static int mem_cgroup_force_empty(struct mem_cgroup *memcg) 3599 { 3600 int nr_retries = MAX_RECLAIM_RETRIES; 3601 3602 /* we call try-to-free pages for make this cgroup empty */ 3603 lru_add_drain_all(); 3604 3605 drain_all_stock(memcg); 3606 3607 /* try to free all pages in this cgroup */ 3608 while (nr_retries && page_counter_read(&memcg->memory)) { 3609 if (signal_pending(current)) 3610 return -EINTR; 3611 3612 if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, 3613 MEMCG_RECLAIM_MAY_SWAP)) 3614 nr_retries--; 3615 } 3616 3617 return 0; 3618 } 3619 3620 static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of, 3621 char *buf, size_t nbytes, 3622 loff_t off) 3623 { 3624 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 3625 3626 if (mem_cgroup_is_root(memcg)) 3627 return -EINVAL; 3628 return mem_cgroup_force_empty(memcg) ?: nbytes; 3629 } 3630 3631 static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css, 3632 struct cftype *cft) 3633 { 3634 return 1; 3635 } 3636 3637 static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css, 3638 struct cftype *cft, u64 val) 3639 { 3640 if (val == 1) 3641 return 0; 3642 3643 pr_warn_once("Non-hierarchical mode is deprecated. " 3644 "Please report your usecase to linux-mm@kvack.org if you " 3645 "depend on this functionality.\n"); 3646 3647 return -EINVAL; 3648 } 3649 3650 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap) 3651 { 3652 unsigned long val; 3653 3654 if (mem_cgroup_is_root(memcg)) { 3655 mem_cgroup_flush_stats(); 3656 val = memcg_page_state(memcg, NR_FILE_PAGES) + 3657 memcg_page_state(memcg, NR_ANON_MAPPED); 3658 if (swap) 3659 val += memcg_page_state(memcg, MEMCG_SWAP); 3660 } else { 3661 if (!swap) 3662 val = page_counter_read(&memcg->memory); 3663 else 3664 val = page_counter_read(&memcg->memsw); 3665 } 3666 return val; 3667 } 3668 3669 enum { 3670 RES_USAGE, 3671 RES_LIMIT, 3672 RES_MAX_USAGE, 3673 RES_FAILCNT, 3674 RES_SOFT_LIMIT, 3675 }; 3676 3677 static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css, 3678 struct cftype *cft) 3679 { 3680 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 3681 struct page_counter *counter; 3682 3683 switch (MEMFILE_TYPE(cft->private)) { 3684 case _MEM: 3685 counter = &memcg->memory; 3686 break; 3687 case _MEMSWAP: 3688 counter = &memcg->memsw; 3689 break; 3690 case _KMEM: 3691 counter = &memcg->kmem; 3692 break; 3693 case _TCP: 3694 counter = &memcg->tcpmem; 3695 break; 3696 default: 3697 BUG(); 3698 } 3699 3700 switch (MEMFILE_ATTR(cft->private)) { 3701 case RES_USAGE: 3702 if (counter == &memcg->memory) 3703 return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE; 3704 if (counter == &memcg->memsw) 3705 return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE; 3706 return (u64)page_counter_read(counter) * PAGE_SIZE; 3707 case RES_LIMIT: 3708 return (u64)counter->max * PAGE_SIZE; 3709 case RES_MAX_USAGE: 3710 return (u64)counter->watermark * PAGE_SIZE; 3711 case RES_FAILCNT: 3712 return counter->failcnt; 3713 case RES_SOFT_LIMIT: 3714 return (u64)memcg->soft_limit * PAGE_SIZE; 3715 default: 3716 BUG(); 3717 } 3718 } 3719 3720 #ifdef CONFIG_MEMCG_KMEM 3721 static int memcg_online_kmem(struct mem_cgroup *memcg) 3722 { 3723 struct obj_cgroup *objcg; 3724 3725 if (mem_cgroup_kmem_disabled()) 3726 return 0; 3727 3728 if (unlikely(mem_cgroup_is_root(memcg))) 3729 return 0; 3730 3731 objcg = obj_cgroup_alloc(); 3732 if (!objcg) 3733 return -ENOMEM; 3734 3735 objcg->memcg = memcg; 3736 rcu_assign_pointer(memcg->objcg, objcg); 3737 3738 static_branch_enable(&memcg_kmem_enabled_key); 3739 3740 memcg->kmemcg_id = memcg->id.id; 3741 3742 return 0; 3743 } 3744 3745 static void memcg_offline_kmem(struct mem_cgroup *memcg) 3746 { 3747 struct mem_cgroup *parent; 3748 3749 if (mem_cgroup_kmem_disabled()) 3750 return; 3751 3752 if (unlikely(mem_cgroup_is_root(memcg))) 3753 return; 3754 3755 parent = parent_mem_cgroup(memcg); 3756 if (!parent) 3757 parent = root_mem_cgroup; 3758 3759 memcg_reparent_objcgs(memcg, parent); 3760 3761 /* 3762 * After we have finished memcg_reparent_objcgs(), all list_lrus 3763 * corresponding to this cgroup are guaranteed to remain empty. 3764 * The ordering is imposed by list_lru_node->lock taken by 3765 * memcg_reparent_list_lrus(). 3766 */ 3767 memcg_reparent_list_lrus(memcg, parent); 3768 } 3769 #else 3770 static int memcg_online_kmem(struct mem_cgroup *memcg) 3771 { 3772 return 0; 3773 } 3774 static void memcg_offline_kmem(struct mem_cgroup *memcg) 3775 { 3776 } 3777 #endif /* CONFIG_MEMCG_KMEM */ 3778 3779 static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max) 3780 { 3781 int ret; 3782 3783 mutex_lock(&memcg_max_mutex); 3784 3785 ret = page_counter_set_max(&memcg->tcpmem, max); 3786 if (ret) 3787 goto out; 3788 3789 if (!memcg->tcpmem_active) { 3790 /* 3791 * The active flag needs to be written after the static_key 3792 * update. This is what guarantees that the socket activation 3793 * function is the last one to run. See mem_cgroup_sk_alloc() 3794 * for details, and note that we don't mark any socket as 3795 * belonging to this memcg until that flag is up. 3796 * 3797 * We need to do this, because static_keys will span multiple 3798 * sites, but we can't control their order. If we mark a socket 3799 * as accounted, but the accounting functions are not patched in 3800 * yet, we'll lose accounting. 3801 * 3802 * We never race with the readers in mem_cgroup_sk_alloc(), 3803 * because when this value change, the code to process it is not 3804 * patched in yet. 3805 */ 3806 static_branch_inc(&memcg_sockets_enabled_key); 3807 memcg->tcpmem_active = true; 3808 } 3809 out: 3810 mutex_unlock(&memcg_max_mutex); 3811 return ret; 3812 } 3813 3814 /* 3815 * The user of this function is... 3816 * RES_LIMIT. 3817 */ 3818 static ssize_t mem_cgroup_write(struct kernfs_open_file *of, 3819 char *buf, size_t nbytes, loff_t off) 3820 { 3821 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 3822 unsigned long nr_pages; 3823 int ret; 3824 3825 buf = strstrip(buf); 3826 ret = page_counter_memparse(buf, "-1", &nr_pages); 3827 if (ret) 3828 return ret; 3829 3830 switch (MEMFILE_ATTR(of_cft(of)->private)) { 3831 case RES_LIMIT: 3832 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */ 3833 ret = -EINVAL; 3834 break; 3835 } 3836 switch (MEMFILE_TYPE(of_cft(of)->private)) { 3837 case _MEM: 3838 ret = mem_cgroup_resize_max(memcg, nr_pages, false); 3839 break; 3840 case _MEMSWAP: 3841 ret = mem_cgroup_resize_max(memcg, nr_pages, true); 3842 break; 3843 case _KMEM: 3844 /* kmem.limit_in_bytes is deprecated. */ 3845 ret = -EOPNOTSUPP; 3846 break; 3847 case _TCP: 3848 ret = memcg_update_tcp_max(memcg, nr_pages); 3849 break; 3850 } 3851 break; 3852 case RES_SOFT_LIMIT: 3853 if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 3854 ret = -EOPNOTSUPP; 3855 } else { 3856 memcg->soft_limit = nr_pages; 3857 ret = 0; 3858 } 3859 break; 3860 } 3861 return ret ?: nbytes; 3862 } 3863 3864 static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf, 3865 size_t nbytes, loff_t off) 3866 { 3867 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 3868 struct page_counter *counter; 3869 3870 switch (MEMFILE_TYPE(of_cft(of)->private)) { 3871 case _MEM: 3872 counter = &memcg->memory; 3873 break; 3874 case _MEMSWAP: 3875 counter = &memcg->memsw; 3876 break; 3877 case _KMEM: 3878 counter = &memcg->kmem; 3879 break; 3880 case _TCP: 3881 counter = &memcg->tcpmem; 3882 break; 3883 default: 3884 BUG(); 3885 } 3886 3887 switch (MEMFILE_ATTR(of_cft(of)->private)) { 3888 case RES_MAX_USAGE: 3889 page_counter_reset_watermark(counter); 3890 break; 3891 case RES_FAILCNT: 3892 counter->failcnt = 0; 3893 break; 3894 default: 3895 BUG(); 3896 } 3897 3898 return nbytes; 3899 } 3900 3901 static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css, 3902 struct cftype *cft) 3903 { 3904 return mem_cgroup_from_css(css)->move_charge_at_immigrate; 3905 } 3906 3907 #ifdef CONFIG_MMU 3908 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css, 3909 struct cftype *cft, u64 val) 3910 { 3911 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 3912 3913 if (val & ~MOVE_MASK) 3914 return -EINVAL; 3915 3916 /* 3917 * No kind of locking is needed in here, because ->can_attach() will 3918 * check this value once in the beginning of the process, and then carry 3919 * on with stale data. This means that changes to this value will only 3920 * affect task migrations starting after the change. 3921 */ 3922 memcg->move_charge_at_immigrate = val; 3923 return 0; 3924 } 3925 #else 3926 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css, 3927 struct cftype *cft, u64 val) 3928 { 3929 return -ENOSYS; 3930 } 3931 #endif 3932 3933 #ifdef CONFIG_NUMA 3934 3935 #define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE)) 3936 #define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON)) 3937 #define LRU_ALL ((1 << NR_LRU_LISTS) - 1) 3938 3939 static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg, 3940 int nid, unsigned int lru_mask, bool tree) 3941 { 3942 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid)); 3943 unsigned long nr = 0; 3944 enum lru_list lru; 3945 3946 VM_BUG_ON((unsigned)nid >= nr_node_ids); 3947 3948 for_each_lru(lru) { 3949 if (!(BIT(lru) & lru_mask)) 3950 continue; 3951 if (tree) 3952 nr += lruvec_page_state(lruvec, NR_LRU_BASE + lru); 3953 else 3954 nr += lruvec_page_state_local(lruvec, NR_LRU_BASE + lru); 3955 } 3956 return nr; 3957 } 3958 3959 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg, 3960 unsigned int lru_mask, 3961 bool tree) 3962 { 3963 unsigned long nr = 0; 3964 enum lru_list lru; 3965 3966 for_each_lru(lru) { 3967 if (!(BIT(lru) & lru_mask)) 3968 continue; 3969 if (tree) 3970 nr += memcg_page_state(memcg, NR_LRU_BASE + lru); 3971 else 3972 nr += memcg_page_state_local(memcg, NR_LRU_BASE + lru); 3973 } 3974 return nr; 3975 } 3976 3977 static int memcg_numa_stat_show(struct seq_file *m, void *v) 3978 { 3979 struct numa_stat { 3980 const char *name; 3981 unsigned int lru_mask; 3982 }; 3983 3984 static const struct numa_stat stats[] = { 3985 { "total", LRU_ALL }, 3986 { "file", LRU_ALL_FILE }, 3987 { "anon", LRU_ALL_ANON }, 3988 { "unevictable", BIT(LRU_UNEVICTABLE) }, 3989 }; 3990 const struct numa_stat *stat; 3991 int nid; 3992 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 3993 3994 mem_cgroup_flush_stats(); 3995 3996 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) { 3997 seq_printf(m, "%s=%lu", stat->name, 3998 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask, 3999 false)); 4000 for_each_node_state(nid, N_MEMORY) 4001 seq_printf(m, " N%d=%lu", nid, 4002 mem_cgroup_node_nr_lru_pages(memcg, nid, 4003 stat->lru_mask, false)); 4004 seq_putc(m, '\n'); 4005 } 4006 4007 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) { 4008 4009 seq_printf(m, "hierarchical_%s=%lu", stat->name, 4010 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask, 4011 true)); 4012 for_each_node_state(nid, N_MEMORY) 4013 seq_printf(m, " N%d=%lu", nid, 4014 mem_cgroup_node_nr_lru_pages(memcg, nid, 4015 stat->lru_mask, true)); 4016 seq_putc(m, '\n'); 4017 } 4018 4019 return 0; 4020 } 4021 #endif /* CONFIG_NUMA */ 4022 4023 static const unsigned int memcg1_stats[] = { 4024 NR_FILE_PAGES, 4025 NR_ANON_MAPPED, 4026 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4027 NR_ANON_THPS, 4028 #endif 4029 NR_SHMEM, 4030 NR_FILE_MAPPED, 4031 NR_FILE_DIRTY, 4032 NR_WRITEBACK, 4033 WORKINGSET_REFAULT_ANON, 4034 WORKINGSET_REFAULT_FILE, 4035 MEMCG_SWAP, 4036 }; 4037 4038 static const char *const memcg1_stat_names[] = { 4039 "cache", 4040 "rss", 4041 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4042 "rss_huge", 4043 #endif 4044 "shmem", 4045 "mapped_file", 4046 "dirty", 4047 "writeback", 4048 "workingset_refault_anon", 4049 "workingset_refault_file", 4050 "swap", 4051 }; 4052 4053 /* Universal VM events cgroup1 shows, original sort order */ 4054 static const unsigned int memcg1_events[] = { 4055 PGPGIN, 4056 PGPGOUT, 4057 PGFAULT, 4058 PGMAJFAULT, 4059 }; 4060 4061 static int memcg_stat_show(struct seq_file *m, void *v) 4062 { 4063 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 4064 unsigned long memory, memsw; 4065 struct mem_cgroup *mi; 4066 unsigned int i; 4067 4068 BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats)); 4069 4070 mem_cgroup_flush_stats(); 4071 4072 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) { 4073 unsigned long nr; 4074 4075 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account()) 4076 continue; 4077 nr = memcg_page_state_local(memcg, memcg1_stats[i]); 4078 seq_printf(m, "%s %lu\n", memcg1_stat_names[i], 4079 nr * memcg_page_state_unit(memcg1_stats[i])); 4080 } 4081 4082 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++) 4083 seq_printf(m, "%s %lu\n", vm_event_name(memcg1_events[i]), 4084 memcg_events_local(memcg, memcg1_events[i])); 4085 4086 for (i = 0; i < NR_LRU_LISTS; i++) 4087 seq_printf(m, "%s %lu\n", lru_list_name(i), 4088 memcg_page_state_local(memcg, NR_LRU_BASE + i) * 4089 PAGE_SIZE); 4090 4091 /* Hierarchical information */ 4092 memory = memsw = PAGE_COUNTER_MAX; 4093 for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) { 4094 memory = min(memory, READ_ONCE(mi->memory.max)); 4095 memsw = min(memsw, READ_ONCE(mi->memsw.max)); 4096 } 4097 seq_printf(m, "hierarchical_memory_limit %llu\n", 4098 (u64)memory * PAGE_SIZE); 4099 if (do_memsw_account()) 4100 seq_printf(m, "hierarchical_memsw_limit %llu\n", 4101 (u64)memsw * PAGE_SIZE); 4102 4103 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) { 4104 unsigned long nr; 4105 4106 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account()) 4107 continue; 4108 nr = memcg_page_state(memcg, memcg1_stats[i]); 4109 seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i], 4110 (u64)nr * memcg_page_state_unit(memcg1_stats[i])); 4111 } 4112 4113 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++) 4114 seq_printf(m, "total_%s %llu\n", 4115 vm_event_name(memcg1_events[i]), 4116 (u64)memcg_events(memcg, memcg1_events[i])); 4117 4118 for (i = 0; i < NR_LRU_LISTS; i++) 4119 seq_printf(m, "total_%s %llu\n", lru_list_name(i), 4120 (u64)memcg_page_state(memcg, NR_LRU_BASE + i) * 4121 PAGE_SIZE); 4122 4123 #ifdef CONFIG_DEBUG_VM 4124 { 4125 pg_data_t *pgdat; 4126 struct mem_cgroup_per_node *mz; 4127 unsigned long anon_cost = 0; 4128 unsigned long file_cost = 0; 4129 4130 for_each_online_pgdat(pgdat) { 4131 mz = memcg->nodeinfo[pgdat->node_id]; 4132 4133 anon_cost += mz->lruvec.anon_cost; 4134 file_cost += mz->lruvec.file_cost; 4135 } 4136 seq_printf(m, "anon_cost %lu\n", anon_cost); 4137 seq_printf(m, "file_cost %lu\n", file_cost); 4138 } 4139 #endif 4140 4141 return 0; 4142 } 4143 4144 static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css, 4145 struct cftype *cft) 4146 { 4147 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4148 4149 return mem_cgroup_swappiness(memcg); 4150 } 4151 4152 static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css, 4153 struct cftype *cft, u64 val) 4154 { 4155 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4156 4157 if (val > 200) 4158 return -EINVAL; 4159 4160 if (!mem_cgroup_is_root(memcg)) 4161 memcg->swappiness = val; 4162 else 4163 vm_swappiness = val; 4164 4165 return 0; 4166 } 4167 4168 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap) 4169 { 4170 struct mem_cgroup_threshold_ary *t; 4171 unsigned long usage; 4172 int i; 4173 4174 rcu_read_lock(); 4175 if (!swap) 4176 t = rcu_dereference(memcg->thresholds.primary); 4177 else 4178 t = rcu_dereference(memcg->memsw_thresholds.primary); 4179 4180 if (!t) 4181 goto unlock; 4182 4183 usage = mem_cgroup_usage(memcg, swap); 4184 4185 /* 4186 * current_threshold points to threshold just below or equal to usage. 4187 * If it's not true, a threshold was crossed after last 4188 * call of __mem_cgroup_threshold(). 4189 */ 4190 i = t->current_threshold; 4191 4192 /* 4193 * Iterate backward over array of thresholds starting from 4194 * current_threshold and check if a threshold is crossed. 4195 * If none of thresholds below usage is crossed, we read 4196 * only one element of the array here. 4197 */ 4198 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--) 4199 eventfd_signal(t->entries[i].eventfd, 1); 4200 4201 /* i = current_threshold + 1 */ 4202 i++; 4203 4204 /* 4205 * Iterate forward over array of thresholds starting from 4206 * current_threshold+1 and check if a threshold is crossed. 4207 * If none of thresholds above usage is crossed, we read 4208 * only one element of the array here. 4209 */ 4210 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++) 4211 eventfd_signal(t->entries[i].eventfd, 1); 4212 4213 /* Update current_threshold */ 4214 t->current_threshold = i - 1; 4215 unlock: 4216 rcu_read_unlock(); 4217 } 4218 4219 static void mem_cgroup_threshold(struct mem_cgroup *memcg) 4220 { 4221 while (memcg) { 4222 __mem_cgroup_threshold(memcg, false); 4223 if (do_memsw_account()) 4224 __mem_cgroup_threshold(memcg, true); 4225 4226 memcg = parent_mem_cgroup(memcg); 4227 } 4228 } 4229 4230 static int compare_thresholds(const void *a, const void *b) 4231 { 4232 const struct mem_cgroup_threshold *_a = a; 4233 const struct mem_cgroup_threshold *_b = b; 4234 4235 if (_a->threshold > _b->threshold) 4236 return 1; 4237 4238 if (_a->threshold < _b->threshold) 4239 return -1; 4240 4241 return 0; 4242 } 4243 4244 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg) 4245 { 4246 struct mem_cgroup_eventfd_list *ev; 4247 4248 spin_lock(&memcg_oom_lock); 4249 4250 list_for_each_entry(ev, &memcg->oom_notify, list) 4251 eventfd_signal(ev->eventfd, 1); 4252 4253 spin_unlock(&memcg_oom_lock); 4254 return 0; 4255 } 4256 4257 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg) 4258 { 4259 struct mem_cgroup *iter; 4260 4261 for_each_mem_cgroup_tree(iter, memcg) 4262 mem_cgroup_oom_notify_cb(iter); 4263 } 4264 4265 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg, 4266 struct eventfd_ctx *eventfd, const char *args, enum res_type type) 4267 { 4268 struct mem_cgroup_thresholds *thresholds; 4269 struct mem_cgroup_threshold_ary *new; 4270 unsigned long threshold; 4271 unsigned long usage; 4272 int i, size, ret; 4273 4274 ret = page_counter_memparse(args, "-1", &threshold); 4275 if (ret) 4276 return ret; 4277 4278 mutex_lock(&memcg->thresholds_lock); 4279 4280 if (type == _MEM) { 4281 thresholds = &memcg->thresholds; 4282 usage = mem_cgroup_usage(memcg, false); 4283 } else if (type == _MEMSWAP) { 4284 thresholds = &memcg->memsw_thresholds; 4285 usage = mem_cgroup_usage(memcg, true); 4286 } else 4287 BUG(); 4288 4289 /* Check if a threshold crossed before adding a new one */ 4290 if (thresholds->primary) 4291 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 4292 4293 size = thresholds->primary ? thresholds->primary->size + 1 : 1; 4294 4295 /* Allocate memory for new array of thresholds */ 4296 new = kmalloc(struct_size(new, entries, size), GFP_KERNEL); 4297 if (!new) { 4298 ret = -ENOMEM; 4299 goto unlock; 4300 } 4301 new->size = size; 4302 4303 /* Copy thresholds (if any) to new array */ 4304 if (thresholds->primary) 4305 memcpy(new->entries, thresholds->primary->entries, 4306 flex_array_size(new, entries, size - 1)); 4307 4308 /* Add new threshold */ 4309 new->entries[size - 1].eventfd = eventfd; 4310 new->entries[size - 1].threshold = threshold; 4311 4312 /* Sort thresholds. Registering of new threshold isn't time-critical */ 4313 sort(new->entries, size, sizeof(*new->entries), 4314 compare_thresholds, NULL); 4315 4316 /* Find current threshold */ 4317 new->current_threshold = -1; 4318 for (i = 0; i < size; i++) { 4319 if (new->entries[i].threshold <= usage) { 4320 /* 4321 * new->current_threshold will not be used until 4322 * rcu_assign_pointer(), so it's safe to increment 4323 * it here. 4324 */ 4325 ++new->current_threshold; 4326 } else 4327 break; 4328 } 4329 4330 /* Free old spare buffer and save old primary buffer as spare */ 4331 kfree(thresholds->spare); 4332 thresholds->spare = thresholds->primary; 4333 4334 rcu_assign_pointer(thresholds->primary, new); 4335 4336 /* To be sure that nobody uses thresholds */ 4337 synchronize_rcu(); 4338 4339 unlock: 4340 mutex_unlock(&memcg->thresholds_lock); 4341 4342 return ret; 4343 } 4344 4345 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg, 4346 struct eventfd_ctx *eventfd, const char *args) 4347 { 4348 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM); 4349 } 4350 4351 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg, 4352 struct eventfd_ctx *eventfd, const char *args) 4353 { 4354 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP); 4355 } 4356 4357 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, 4358 struct eventfd_ctx *eventfd, enum res_type type) 4359 { 4360 struct mem_cgroup_thresholds *thresholds; 4361 struct mem_cgroup_threshold_ary *new; 4362 unsigned long usage; 4363 int i, j, size, entries; 4364 4365 mutex_lock(&memcg->thresholds_lock); 4366 4367 if (type == _MEM) { 4368 thresholds = &memcg->thresholds; 4369 usage = mem_cgroup_usage(memcg, false); 4370 } else if (type == _MEMSWAP) { 4371 thresholds = &memcg->memsw_thresholds; 4372 usage = mem_cgroup_usage(memcg, true); 4373 } else 4374 BUG(); 4375 4376 if (!thresholds->primary) 4377 goto unlock; 4378 4379 /* Check if a threshold crossed before removing */ 4380 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 4381 4382 /* Calculate new number of threshold */ 4383 size = entries = 0; 4384 for (i = 0; i < thresholds->primary->size; i++) { 4385 if (thresholds->primary->entries[i].eventfd != eventfd) 4386 size++; 4387 else 4388 entries++; 4389 } 4390 4391 new = thresholds->spare; 4392 4393 /* If no items related to eventfd have been cleared, nothing to do */ 4394 if (!entries) 4395 goto unlock; 4396 4397 /* Set thresholds array to NULL if we don't have thresholds */ 4398 if (!size) { 4399 kfree(new); 4400 new = NULL; 4401 goto swap_buffers; 4402 } 4403 4404 new->size = size; 4405 4406 /* Copy thresholds and find current threshold */ 4407 new->current_threshold = -1; 4408 for (i = 0, j = 0; i < thresholds->primary->size; i++) { 4409 if (thresholds->primary->entries[i].eventfd == eventfd) 4410 continue; 4411 4412 new->entries[j] = thresholds->primary->entries[i]; 4413 if (new->entries[j].threshold <= usage) { 4414 /* 4415 * new->current_threshold will not be used 4416 * until rcu_assign_pointer(), so it's safe to increment 4417 * it here. 4418 */ 4419 ++new->current_threshold; 4420 } 4421 j++; 4422 } 4423 4424 swap_buffers: 4425 /* Swap primary and spare array */ 4426 thresholds->spare = thresholds->primary; 4427 4428 rcu_assign_pointer(thresholds->primary, new); 4429 4430 /* To be sure that nobody uses thresholds */ 4431 synchronize_rcu(); 4432 4433 /* If all events are unregistered, free the spare array */ 4434 if (!new) { 4435 kfree(thresholds->spare); 4436 thresholds->spare = NULL; 4437 } 4438 unlock: 4439 mutex_unlock(&memcg->thresholds_lock); 4440 } 4441 4442 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, 4443 struct eventfd_ctx *eventfd) 4444 { 4445 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM); 4446 } 4447 4448 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg, 4449 struct eventfd_ctx *eventfd) 4450 { 4451 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP); 4452 } 4453 4454 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg, 4455 struct eventfd_ctx *eventfd, const char *args) 4456 { 4457 struct mem_cgroup_eventfd_list *event; 4458 4459 event = kmalloc(sizeof(*event), GFP_KERNEL); 4460 if (!event) 4461 return -ENOMEM; 4462 4463 spin_lock(&memcg_oom_lock); 4464 4465 event->eventfd = eventfd; 4466 list_add(&event->list, &memcg->oom_notify); 4467 4468 /* already in OOM ? */ 4469 if (memcg->under_oom) 4470 eventfd_signal(eventfd, 1); 4471 spin_unlock(&memcg_oom_lock); 4472 4473 return 0; 4474 } 4475 4476 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg, 4477 struct eventfd_ctx *eventfd) 4478 { 4479 struct mem_cgroup_eventfd_list *ev, *tmp; 4480 4481 spin_lock(&memcg_oom_lock); 4482 4483 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) { 4484 if (ev->eventfd == eventfd) { 4485 list_del(&ev->list); 4486 kfree(ev); 4487 } 4488 } 4489 4490 spin_unlock(&memcg_oom_lock); 4491 } 4492 4493 static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v) 4494 { 4495 struct mem_cgroup *memcg = mem_cgroup_from_seq(sf); 4496 4497 seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable); 4498 seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom); 4499 seq_printf(sf, "oom_kill %lu\n", 4500 atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL])); 4501 return 0; 4502 } 4503 4504 static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css, 4505 struct cftype *cft, u64 val) 4506 { 4507 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4508 4509 /* cannot set to root cgroup and only 0 and 1 are allowed */ 4510 if (mem_cgroup_is_root(memcg) || !((val == 0) || (val == 1))) 4511 return -EINVAL; 4512 4513 memcg->oom_kill_disable = val; 4514 if (!val) 4515 memcg_oom_recover(memcg); 4516 4517 return 0; 4518 } 4519 4520 #ifdef CONFIG_CGROUP_WRITEBACK 4521 4522 #include <trace/events/writeback.h> 4523 4524 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp) 4525 { 4526 return wb_domain_init(&memcg->cgwb_domain, gfp); 4527 } 4528 4529 static void memcg_wb_domain_exit(struct mem_cgroup *memcg) 4530 { 4531 wb_domain_exit(&memcg->cgwb_domain); 4532 } 4533 4534 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg) 4535 { 4536 wb_domain_size_changed(&memcg->cgwb_domain); 4537 } 4538 4539 struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb) 4540 { 4541 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); 4542 4543 if (!memcg->css.parent) 4544 return NULL; 4545 4546 return &memcg->cgwb_domain; 4547 } 4548 4549 /** 4550 * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg 4551 * @wb: bdi_writeback in question 4552 * @pfilepages: out parameter for number of file pages 4553 * @pheadroom: out parameter for number of allocatable pages according to memcg 4554 * @pdirty: out parameter for number of dirty pages 4555 * @pwriteback: out parameter for number of pages under writeback 4556 * 4557 * Determine the numbers of file, headroom, dirty, and writeback pages in 4558 * @wb's memcg. File, dirty and writeback are self-explanatory. Headroom 4559 * is a bit more involved. 4560 * 4561 * A memcg's headroom is "min(max, high) - used". In the hierarchy, the 4562 * headroom is calculated as the lowest headroom of itself and the 4563 * ancestors. Note that this doesn't consider the actual amount of 4564 * available memory in the system. The caller should further cap 4565 * *@pheadroom accordingly. 4566 */ 4567 void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages, 4568 unsigned long *pheadroom, unsigned long *pdirty, 4569 unsigned long *pwriteback) 4570 { 4571 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); 4572 struct mem_cgroup *parent; 4573 4574 mem_cgroup_flush_stats(); 4575 4576 *pdirty = memcg_page_state(memcg, NR_FILE_DIRTY); 4577 *pwriteback = memcg_page_state(memcg, NR_WRITEBACK); 4578 *pfilepages = memcg_page_state(memcg, NR_INACTIVE_FILE) + 4579 memcg_page_state(memcg, NR_ACTIVE_FILE); 4580 4581 *pheadroom = PAGE_COUNTER_MAX; 4582 while ((parent = parent_mem_cgroup(memcg))) { 4583 unsigned long ceiling = min(READ_ONCE(memcg->memory.max), 4584 READ_ONCE(memcg->memory.high)); 4585 unsigned long used = page_counter_read(&memcg->memory); 4586 4587 *pheadroom = min(*pheadroom, ceiling - min(ceiling, used)); 4588 memcg = parent; 4589 } 4590 } 4591 4592 /* 4593 * Foreign dirty flushing 4594 * 4595 * There's an inherent mismatch between memcg and writeback. The former 4596 * tracks ownership per-page while the latter per-inode. This was a 4597 * deliberate design decision because honoring per-page ownership in the 4598 * writeback path is complicated, may lead to higher CPU and IO overheads 4599 * and deemed unnecessary given that write-sharing an inode across 4600 * different cgroups isn't a common use-case. 4601 * 4602 * Combined with inode majority-writer ownership switching, this works well 4603 * enough in most cases but there are some pathological cases. For 4604 * example, let's say there are two cgroups A and B which keep writing to 4605 * different but confined parts of the same inode. B owns the inode and 4606 * A's memory is limited far below B's. A's dirty ratio can rise enough to 4607 * trigger balance_dirty_pages() sleeps but B's can be low enough to avoid 4608 * triggering background writeback. A will be slowed down without a way to 4609 * make writeback of the dirty pages happen. 4610 * 4611 * Conditions like the above can lead to a cgroup getting repeatedly and 4612 * severely throttled after making some progress after each 4613 * dirty_expire_interval while the underlying IO device is almost 4614 * completely idle. 4615 * 4616 * Solving this problem completely requires matching the ownership tracking 4617 * granularities between memcg and writeback in either direction. However, 4618 * the more egregious behaviors can be avoided by simply remembering the 4619 * most recent foreign dirtying events and initiating remote flushes on 4620 * them when local writeback isn't enough to keep the memory clean enough. 4621 * 4622 * The following two functions implement such mechanism. When a foreign 4623 * page - a page whose memcg and writeback ownerships don't match - is 4624 * dirtied, mem_cgroup_track_foreign_dirty() records the inode owning 4625 * bdi_writeback on the page owning memcg. When balance_dirty_pages() 4626 * decides that the memcg needs to sleep due to high dirty ratio, it calls 4627 * mem_cgroup_flush_foreign() which queues writeback on the recorded 4628 * foreign bdi_writebacks which haven't expired. Both the numbers of 4629 * recorded bdi_writebacks and concurrent in-flight foreign writebacks are 4630 * limited to MEMCG_CGWB_FRN_CNT. 4631 * 4632 * The mechanism only remembers IDs and doesn't hold any object references. 4633 * As being wrong occasionally doesn't matter, updates and accesses to the 4634 * records are lockless and racy. 4635 */ 4636 void mem_cgroup_track_foreign_dirty_slowpath(struct folio *folio, 4637 struct bdi_writeback *wb) 4638 { 4639 struct mem_cgroup *memcg = folio_memcg(folio); 4640 struct memcg_cgwb_frn *frn; 4641 u64 now = get_jiffies_64(); 4642 u64 oldest_at = now; 4643 int oldest = -1; 4644 int i; 4645 4646 trace_track_foreign_dirty(folio, wb); 4647 4648 /* 4649 * Pick the slot to use. If there is already a slot for @wb, keep 4650 * using it. If not replace the oldest one which isn't being 4651 * written out. 4652 */ 4653 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) { 4654 frn = &memcg->cgwb_frn[i]; 4655 if (frn->bdi_id == wb->bdi->id && 4656 frn->memcg_id == wb->memcg_css->id) 4657 break; 4658 if (time_before64(frn->at, oldest_at) && 4659 atomic_read(&frn->done.cnt) == 1) { 4660 oldest = i; 4661 oldest_at = frn->at; 4662 } 4663 } 4664 4665 if (i < MEMCG_CGWB_FRN_CNT) { 4666 /* 4667 * Re-using an existing one. Update timestamp lazily to 4668 * avoid making the cacheline hot. We want them to be 4669 * reasonably up-to-date and significantly shorter than 4670 * dirty_expire_interval as that's what expires the record. 4671 * Use the shorter of 1s and dirty_expire_interval / 8. 4672 */ 4673 unsigned long update_intv = 4674 min_t(unsigned long, HZ, 4675 msecs_to_jiffies(dirty_expire_interval * 10) / 8); 4676 4677 if (time_before64(frn->at, now - update_intv)) 4678 frn->at = now; 4679 } else if (oldest >= 0) { 4680 /* replace the oldest free one */ 4681 frn = &memcg->cgwb_frn[oldest]; 4682 frn->bdi_id = wb->bdi->id; 4683 frn->memcg_id = wb->memcg_css->id; 4684 frn->at = now; 4685 } 4686 } 4687 4688 /* issue foreign writeback flushes for recorded foreign dirtying events */ 4689 void mem_cgroup_flush_foreign(struct bdi_writeback *wb) 4690 { 4691 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); 4692 unsigned long intv = msecs_to_jiffies(dirty_expire_interval * 10); 4693 u64 now = jiffies_64; 4694 int i; 4695 4696 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) { 4697 struct memcg_cgwb_frn *frn = &memcg->cgwb_frn[i]; 4698 4699 /* 4700 * If the record is older than dirty_expire_interval, 4701 * writeback on it has already started. No need to kick it 4702 * off again. Also, don't start a new one if there's 4703 * already one in flight. 4704 */ 4705 if (time_after64(frn->at, now - intv) && 4706 atomic_read(&frn->done.cnt) == 1) { 4707 frn->at = 0; 4708 trace_flush_foreign(wb, frn->bdi_id, frn->memcg_id); 4709 cgroup_writeback_by_id(frn->bdi_id, frn->memcg_id, 4710 WB_REASON_FOREIGN_FLUSH, 4711 &frn->done); 4712 } 4713 } 4714 } 4715 4716 #else /* CONFIG_CGROUP_WRITEBACK */ 4717 4718 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp) 4719 { 4720 return 0; 4721 } 4722 4723 static void memcg_wb_domain_exit(struct mem_cgroup *memcg) 4724 { 4725 } 4726 4727 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg) 4728 { 4729 } 4730 4731 #endif /* CONFIG_CGROUP_WRITEBACK */ 4732 4733 /* 4734 * DO NOT USE IN NEW FILES. 4735 * 4736 * "cgroup.event_control" implementation. 4737 * 4738 * This is way over-engineered. It tries to support fully configurable 4739 * events for each user. Such level of flexibility is completely 4740 * unnecessary especially in the light of the planned unified hierarchy. 4741 * 4742 * Please deprecate this and replace with something simpler if at all 4743 * possible. 4744 */ 4745 4746 /* 4747 * Unregister event and free resources. 4748 * 4749 * Gets called from workqueue. 4750 */ 4751 static void memcg_event_remove(struct work_struct *work) 4752 { 4753 struct mem_cgroup_event *event = 4754 container_of(work, struct mem_cgroup_event, remove); 4755 struct mem_cgroup *memcg = event->memcg; 4756 4757 remove_wait_queue(event->wqh, &event->wait); 4758 4759 event->unregister_event(memcg, event->eventfd); 4760 4761 /* Notify userspace the event is going away. */ 4762 eventfd_signal(event->eventfd, 1); 4763 4764 eventfd_ctx_put(event->eventfd); 4765 kfree(event); 4766 css_put(&memcg->css); 4767 } 4768 4769 /* 4770 * Gets called on EPOLLHUP on eventfd when user closes it. 4771 * 4772 * Called with wqh->lock held and interrupts disabled. 4773 */ 4774 static int memcg_event_wake(wait_queue_entry_t *wait, unsigned mode, 4775 int sync, void *key) 4776 { 4777 struct mem_cgroup_event *event = 4778 container_of(wait, struct mem_cgroup_event, wait); 4779 struct mem_cgroup *memcg = event->memcg; 4780 __poll_t flags = key_to_poll(key); 4781 4782 if (flags & EPOLLHUP) { 4783 /* 4784 * If the event has been detached at cgroup removal, we 4785 * can simply return knowing the other side will cleanup 4786 * for us. 4787 * 4788 * We can't race against event freeing since the other 4789 * side will require wqh->lock via remove_wait_queue(), 4790 * which we hold. 4791 */ 4792 spin_lock(&memcg->event_list_lock); 4793 if (!list_empty(&event->list)) { 4794 list_del_init(&event->list); 4795 /* 4796 * We are in atomic context, but cgroup_event_remove() 4797 * may sleep, so we have to call it in workqueue. 4798 */ 4799 schedule_work(&event->remove); 4800 } 4801 spin_unlock(&memcg->event_list_lock); 4802 } 4803 4804 return 0; 4805 } 4806 4807 static void memcg_event_ptable_queue_proc(struct file *file, 4808 wait_queue_head_t *wqh, poll_table *pt) 4809 { 4810 struct mem_cgroup_event *event = 4811 container_of(pt, struct mem_cgroup_event, pt); 4812 4813 event->wqh = wqh; 4814 add_wait_queue(wqh, &event->wait); 4815 } 4816 4817 /* 4818 * DO NOT USE IN NEW FILES. 4819 * 4820 * Parse input and register new cgroup event handler. 4821 * 4822 * Input must be in format '<event_fd> <control_fd> <args>'. 4823 * Interpretation of args is defined by control file implementation. 4824 */ 4825 static ssize_t memcg_write_event_control(struct kernfs_open_file *of, 4826 char *buf, size_t nbytes, loff_t off) 4827 { 4828 struct cgroup_subsys_state *css = of_css(of); 4829 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4830 struct mem_cgroup_event *event; 4831 struct cgroup_subsys_state *cfile_css; 4832 unsigned int efd, cfd; 4833 struct fd efile; 4834 struct fd cfile; 4835 const char *name; 4836 char *endp; 4837 int ret; 4838 4839 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 4840 return -EOPNOTSUPP; 4841 4842 buf = strstrip(buf); 4843 4844 efd = simple_strtoul(buf, &endp, 10); 4845 if (*endp != ' ') 4846 return -EINVAL; 4847 buf = endp + 1; 4848 4849 cfd = simple_strtoul(buf, &endp, 10); 4850 if ((*endp != ' ') && (*endp != '\0')) 4851 return -EINVAL; 4852 buf = endp + 1; 4853 4854 event = kzalloc(sizeof(*event), GFP_KERNEL); 4855 if (!event) 4856 return -ENOMEM; 4857 4858 event->memcg = memcg; 4859 INIT_LIST_HEAD(&event->list); 4860 init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc); 4861 init_waitqueue_func_entry(&event->wait, memcg_event_wake); 4862 INIT_WORK(&event->remove, memcg_event_remove); 4863 4864 efile = fdget(efd); 4865 if (!efile.file) { 4866 ret = -EBADF; 4867 goto out_kfree; 4868 } 4869 4870 event->eventfd = eventfd_ctx_fileget(efile.file); 4871 if (IS_ERR(event->eventfd)) { 4872 ret = PTR_ERR(event->eventfd); 4873 goto out_put_efile; 4874 } 4875 4876 cfile = fdget(cfd); 4877 if (!cfile.file) { 4878 ret = -EBADF; 4879 goto out_put_eventfd; 4880 } 4881 4882 /* the process need read permission on control file */ 4883 /* AV: shouldn't we check that it's been opened for read instead? */ 4884 ret = file_permission(cfile.file, MAY_READ); 4885 if (ret < 0) 4886 goto out_put_cfile; 4887 4888 /* 4889 * Determine the event callbacks and set them in @event. This used 4890 * to be done via struct cftype but cgroup core no longer knows 4891 * about these events. The following is crude but the whole thing 4892 * is for compatibility anyway. 4893 * 4894 * DO NOT ADD NEW FILES. 4895 */ 4896 name = cfile.file->f_path.dentry->d_name.name; 4897 4898 if (!strcmp(name, "memory.usage_in_bytes")) { 4899 event->register_event = mem_cgroup_usage_register_event; 4900 event->unregister_event = mem_cgroup_usage_unregister_event; 4901 } else if (!strcmp(name, "memory.oom_control")) { 4902 event->register_event = mem_cgroup_oom_register_event; 4903 event->unregister_event = mem_cgroup_oom_unregister_event; 4904 } else if (!strcmp(name, "memory.pressure_level")) { 4905 event->register_event = vmpressure_register_event; 4906 event->unregister_event = vmpressure_unregister_event; 4907 } else if (!strcmp(name, "memory.memsw.usage_in_bytes")) { 4908 event->register_event = memsw_cgroup_usage_register_event; 4909 event->unregister_event = memsw_cgroup_usage_unregister_event; 4910 } else { 4911 ret = -EINVAL; 4912 goto out_put_cfile; 4913 } 4914 4915 /* 4916 * Verify @cfile should belong to @css. Also, remaining events are 4917 * automatically removed on cgroup destruction but the removal is 4918 * asynchronous, so take an extra ref on @css. 4919 */ 4920 cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent, 4921 &memory_cgrp_subsys); 4922 ret = -EINVAL; 4923 if (IS_ERR(cfile_css)) 4924 goto out_put_cfile; 4925 if (cfile_css != css) { 4926 css_put(cfile_css); 4927 goto out_put_cfile; 4928 } 4929 4930 ret = event->register_event(memcg, event->eventfd, buf); 4931 if (ret) 4932 goto out_put_css; 4933 4934 vfs_poll(efile.file, &event->pt); 4935 4936 spin_lock_irq(&memcg->event_list_lock); 4937 list_add(&event->list, &memcg->event_list); 4938 spin_unlock_irq(&memcg->event_list_lock); 4939 4940 fdput(cfile); 4941 fdput(efile); 4942 4943 return nbytes; 4944 4945 out_put_css: 4946 css_put(css); 4947 out_put_cfile: 4948 fdput(cfile); 4949 out_put_eventfd: 4950 eventfd_ctx_put(event->eventfd); 4951 out_put_efile: 4952 fdput(efile); 4953 out_kfree: 4954 kfree(event); 4955 4956 return ret; 4957 } 4958 4959 #if defined(CONFIG_MEMCG_KMEM) && (defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)) 4960 static int mem_cgroup_slab_show(struct seq_file *m, void *p) 4961 { 4962 /* 4963 * Deprecated. 4964 * Please, take a look at tools/cgroup/memcg_slabinfo.py . 4965 */ 4966 return 0; 4967 } 4968 #endif 4969 4970 static struct cftype mem_cgroup_legacy_files[] = { 4971 { 4972 .name = "usage_in_bytes", 4973 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE), 4974 .read_u64 = mem_cgroup_read_u64, 4975 }, 4976 { 4977 .name = "max_usage_in_bytes", 4978 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE), 4979 .write = mem_cgroup_reset, 4980 .read_u64 = mem_cgroup_read_u64, 4981 }, 4982 { 4983 .name = "limit_in_bytes", 4984 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT), 4985 .write = mem_cgroup_write, 4986 .read_u64 = mem_cgroup_read_u64, 4987 }, 4988 { 4989 .name = "soft_limit_in_bytes", 4990 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT), 4991 .write = mem_cgroup_write, 4992 .read_u64 = mem_cgroup_read_u64, 4993 }, 4994 { 4995 .name = "failcnt", 4996 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT), 4997 .write = mem_cgroup_reset, 4998 .read_u64 = mem_cgroup_read_u64, 4999 }, 5000 { 5001 .name = "stat", 5002 .seq_show = memcg_stat_show, 5003 }, 5004 { 5005 .name = "force_empty", 5006 .write = mem_cgroup_force_empty_write, 5007 }, 5008 { 5009 .name = "use_hierarchy", 5010 .write_u64 = mem_cgroup_hierarchy_write, 5011 .read_u64 = mem_cgroup_hierarchy_read, 5012 }, 5013 { 5014 .name = "cgroup.event_control", /* XXX: for compat */ 5015 .write = memcg_write_event_control, 5016 .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE, 5017 }, 5018 { 5019 .name = "swappiness", 5020 .read_u64 = mem_cgroup_swappiness_read, 5021 .write_u64 = mem_cgroup_swappiness_write, 5022 }, 5023 { 5024 .name = "move_charge_at_immigrate", 5025 .read_u64 = mem_cgroup_move_charge_read, 5026 .write_u64 = mem_cgroup_move_charge_write, 5027 }, 5028 { 5029 .name = "oom_control", 5030 .seq_show = mem_cgroup_oom_control_read, 5031 .write_u64 = mem_cgroup_oom_control_write, 5032 }, 5033 { 5034 .name = "pressure_level", 5035 }, 5036 #ifdef CONFIG_NUMA 5037 { 5038 .name = "numa_stat", 5039 .seq_show = memcg_numa_stat_show, 5040 }, 5041 #endif 5042 { 5043 .name = "kmem.limit_in_bytes", 5044 .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT), 5045 .write = mem_cgroup_write, 5046 .read_u64 = mem_cgroup_read_u64, 5047 }, 5048 { 5049 .name = "kmem.usage_in_bytes", 5050 .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE), 5051 .read_u64 = mem_cgroup_read_u64, 5052 }, 5053 { 5054 .name = "kmem.failcnt", 5055 .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT), 5056 .write = mem_cgroup_reset, 5057 .read_u64 = mem_cgroup_read_u64, 5058 }, 5059 { 5060 .name = "kmem.max_usage_in_bytes", 5061 .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE), 5062 .write = mem_cgroup_reset, 5063 .read_u64 = mem_cgroup_read_u64, 5064 }, 5065 #if defined(CONFIG_MEMCG_KMEM) && \ 5066 (defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)) 5067 { 5068 .name = "kmem.slabinfo", 5069 .seq_show = mem_cgroup_slab_show, 5070 }, 5071 #endif 5072 { 5073 .name = "kmem.tcp.limit_in_bytes", 5074 .private = MEMFILE_PRIVATE(_TCP, RES_LIMIT), 5075 .write = mem_cgroup_write, 5076 .read_u64 = mem_cgroup_read_u64, 5077 }, 5078 { 5079 .name = "kmem.tcp.usage_in_bytes", 5080 .private = MEMFILE_PRIVATE(_TCP, RES_USAGE), 5081 .read_u64 = mem_cgroup_read_u64, 5082 }, 5083 { 5084 .name = "kmem.tcp.failcnt", 5085 .private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT), 5086 .write = mem_cgroup_reset, 5087 .read_u64 = mem_cgroup_read_u64, 5088 }, 5089 { 5090 .name = "kmem.tcp.max_usage_in_bytes", 5091 .private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE), 5092 .write = mem_cgroup_reset, 5093 .read_u64 = mem_cgroup_read_u64, 5094 }, 5095 { }, /* terminate */ 5096 }; 5097 5098 /* 5099 * Private memory cgroup IDR 5100 * 5101 * Swap-out records and page cache shadow entries need to store memcg 5102 * references in constrained space, so we maintain an ID space that is 5103 * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of 5104 * memory-controlled cgroups to 64k. 5105 * 5106 * However, there usually are many references to the offline CSS after 5107 * the cgroup has been destroyed, such as page cache or reclaimable 5108 * slab objects, that don't need to hang on to the ID. We want to keep 5109 * those dead CSS from occupying IDs, or we might quickly exhaust the 5110 * relatively small ID space and prevent the creation of new cgroups 5111 * even when there are much fewer than 64k cgroups - possibly none. 5112 * 5113 * Maintain a private 16-bit ID space for memcg, and allow the ID to 5114 * be freed and recycled when it's no longer needed, which is usually 5115 * when the CSS is offlined. 5116 * 5117 * The only exception to that are records of swapped out tmpfs/shmem 5118 * pages that need to be attributed to live ancestors on swapin. But 5119 * those references are manageable from userspace. 5120 */ 5121 5122 static DEFINE_IDR(mem_cgroup_idr); 5123 5124 static void mem_cgroup_id_remove(struct mem_cgroup *memcg) 5125 { 5126 if (memcg->id.id > 0) { 5127 idr_remove(&mem_cgroup_idr, memcg->id.id); 5128 memcg->id.id = 0; 5129 } 5130 } 5131 5132 static void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg, 5133 unsigned int n) 5134 { 5135 refcount_add(n, &memcg->id.ref); 5136 } 5137 5138 static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n) 5139 { 5140 if (refcount_sub_and_test(n, &memcg->id.ref)) { 5141 mem_cgroup_id_remove(memcg); 5142 5143 /* Memcg ID pins CSS */ 5144 css_put(&memcg->css); 5145 } 5146 } 5147 5148 static inline void mem_cgroup_id_put(struct mem_cgroup *memcg) 5149 { 5150 mem_cgroup_id_put_many(memcg, 1); 5151 } 5152 5153 /** 5154 * mem_cgroup_from_id - look up a memcg from a memcg id 5155 * @id: the memcg id to look up 5156 * 5157 * Caller must hold rcu_read_lock(). 5158 */ 5159 struct mem_cgroup *mem_cgroup_from_id(unsigned short id) 5160 { 5161 WARN_ON_ONCE(!rcu_read_lock_held()); 5162 return idr_find(&mem_cgroup_idr, id); 5163 } 5164 5165 #ifdef CONFIG_SHRINKER_DEBUG 5166 struct mem_cgroup *mem_cgroup_get_from_ino(unsigned long ino) 5167 { 5168 struct cgroup *cgrp; 5169 struct cgroup_subsys_state *css; 5170 struct mem_cgroup *memcg; 5171 5172 cgrp = cgroup_get_from_id(ino); 5173 if (IS_ERR(cgrp)) 5174 return ERR_CAST(cgrp); 5175 5176 css = cgroup_get_e_css(cgrp, &memory_cgrp_subsys); 5177 if (css) 5178 memcg = container_of(css, struct mem_cgroup, css); 5179 else 5180 memcg = ERR_PTR(-ENOENT); 5181 5182 cgroup_put(cgrp); 5183 5184 return memcg; 5185 } 5186 #endif 5187 5188 static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) 5189 { 5190 struct mem_cgroup_per_node *pn; 5191 5192 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, node); 5193 if (!pn) 5194 return 1; 5195 5196 pn->lruvec_stats_percpu = alloc_percpu_gfp(struct lruvec_stats_percpu, 5197 GFP_KERNEL_ACCOUNT); 5198 if (!pn->lruvec_stats_percpu) { 5199 kfree(pn); 5200 return 1; 5201 } 5202 5203 lruvec_init(&pn->lruvec); 5204 pn->memcg = memcg; 5205 5206 memcg->nodeinfo[node] = pn; 5207 return 0; 5208 } 5209 5210 static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) 5211 { 5212 struct mem_cgroup_per_node *pn = memcg->nodeinfo[node]; 5213 5214 if (!pn) 5215 return; 5216 5217 free_percpu(pn->lruvec_stats_percpu); 5218 kfree(pn); 5219 } 5220 5221 static void __mem_cgroup_free(struct mem_cgroup *memcg) 5222 { 5223 int node; 5224 5225 for_each_node(node) 5226 free_mem_cgroup_per_node_info(memcg, node); 5227 kfree(memcg->vmstats); 5228 free_percpu(memcg->vmstats_percpu); 5229 kfree(memcg); 5230 } 5231 5232 static void mem_cgroup_free(struct mem_cgroup *memcg) 5233 { 5234 lru_gen_exit_memcg(memcg); 5235 memcg_wb_domain_exit(memcg); 5236 __mem_cgroup_free(memcg); 5237 } 5238 5239 static struct mem_cgroup *mem_cgroup_alloc(void) 5240 { 5241 struct mem_cgroup *memcg; 5242 int node; 5243 int __maybe_unused i; 5244 long error = -ENOMEM; 5245 5246 memcg = kzalloc(struct_size(memcg, nodeinfo, nr_node_ids), GFP_KERNEL); 5247 if (!memcg) 5248 return ERR_PTR(error); 5249 5250 memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL, 5251 1, MEM_CGROUP_ID_MAX + 1, GFP_KERNEL); 5252 if (memcg->id.id < 0) { 5253 error = memcg->id.id; 5254 goto fail; 5255 } 5256 5257 memcg->vmstats = kzalloc(sizeof(struct memcg_vmstats), GFP_KERNEL); 5258 if (!memcg->vmstats) 5259 goto fail; 5260 5261 memcg->vmstats_percpu = alloc_percpu_gfp(struct memcg_vmstats_percpu, 5262 GFP_KERNEL_ACCOUNT); 5263 if (!memcg->vmstats_percpu) 5264 goto fail; 5265 5266 for_each_node(node) 5267 if (alloc_mem_cgroup_per_node_info(memcg, node)) 5268 goto fail; 5269 5270 if (memcg_wb_domain_init(memcg, GFP_KERNEL)) 5271 goto fail; 5272 5273 INIT_WORK(&memcg->high_work, high_work_func); 5274 INIT_LIST_HEAD(&memcg->oom_notify); 5275 mutex_init(&memcg->thresholds_lock); 5276 spin_lock_init(&memcg->move_lock); 5277 vmpressure_init(&memcg->vmpressure); 5278 INIT_LIST_HEAD(&memcg->event_list); 5279 spin_lock_init(&memcg->event_list_lock); 5280 memcg->socket_pressure = jiffies; 5281 #ifdef CONFIG_MEMCG_KMEM 5282 memcg->kmemcg_id = -1; 5283 INIT_LIST_HEAD(&memcg->objcg_list); 5284 #endif 5285 #ifdef CONFIG_CGROUP_WRITEBACK 5286 INIT_LIST_HEAD(&memcg->cgwb_list); 5287 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) 5288 memcg->cgwb_frn[i].done = 5289 __WB_COMPLETION_INIT(&memcg_cgwb_frn_waitq); 5290 #endif 5291 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5292 spin_lock_init(&memcg->deferred_split_queue.split_queue_lock); 5293 INIT_LIST_HEAD(&memcg->deferred_split_queue.split_queue); 5294 memcg->deferred_split_queue.split_queue_len = 0; 5295 #endif 5296 idr_replace(&mem_cgroup_idr, memcg, memcg->id.id); 5297 lru_gen_init_memcg(memcg); 5298 return memcg; 5299 fail: 5300 mem_cgroup_id_remove(memcg); 5301 __mem_cgroup_free(memcg); 5302 return ERR_PTR(error); 5303 } 5304 5305 static struct cgroup_subsys_state * __ref 5306 mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 5307 { 5308 struct mem_cgroup *parent = mem_cgroup_from_css(parent_css); 5309 struct mem_cgroup *memcg, *old_memcg; 5310 5311 old_memcg = set_active_memcg(parent); 5312 memcg = mem_cgroup_alloc(); 5313 set_active_memcg(old_memcg); 5314 if (IS_ERR(memcg)) 5315 return ERR_CAST(memcg); 5316 5317 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX); 5318 memcg->soft_limit = PAGE_COUNTER_MAX; 5319 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP) 5320 memcg->zswap_max = PAGE_COUNTER_MAX; 5321 #endif 5322 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX); 5323 if (parent) { 5324 memcg->swappiness = mem_cgroup_swappiness(parent); 5325 memcg->oom_kill_disable = parent->oom_kill_disable; 5326 5327 page_counter_init(&memcg->memory, &parent->memory); 5328 page_counter_init(&memcg->swap, &parent->swap); 5329 page_counter_init(&memcg->kmem, &parent->kmem); 5330 page_counter_init(&memcg->tcpmem, &parent->tcpmem); 5331 } else { 5332 init_memcg_events(); 5333 page_counter_init(&memcg->memory, NULL); 5334 page_counter_init(&memcg->swap, NULL); 5335 page_counter_init(&memcg->kmem, NULL); 5336 page_counter_init(&memcg->tcpmem, NULL); 5337 5338 root_mem_cgroup = memcg; 5339 return &memcg->css; 5340 } 5341 5342 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket) 5343 static_branch_inc(&memcg_sockets_enabled_key); 5344 5345 return &memcg->css; 5346 } 5347 5348 static int mem_cgroup_css_online(struct cgroup_subsys_state *css) 5349 { 5350 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5351 5352 if (memcg_online_kmem(memcg)) 5353 goto remove_id; 5354 5355 /* 5356 * A memcg must be visible for expand_shrinker_info() 5357 * by the time the maps are allocated. So, we allocate maps 5358 * here, when for_each_mem_cgroup() can't skip it. 5359 */ 5360 if (alloc_shrinker_info(memcg)) 5361 goto offline_kmem; 5362 5363 /* Online state pins memcg ID, memcg ID pins CSS */ 5364 refcount_set(&memcg->id.ref, 1); 5365 css_get(css); 5366 5367 if (unlikely(mem_cgroup_is_root(memcg))) 5368 queue_delayed_work(system_unbound_wq, &stats_flush_dwork, 5369 2UL*HZ); 5370 return 0; 5371 offline_kmem: 5372 memcg_offline_kmem(memcg); 5373 remove_id: 5374 mem_cgroup_id_remove(memcg); 5375 return -ENOMEM; 5376 } 5377 5378 static void mem_cgroup_css_offline(struct cgroup_subsys_state *css) 5379 { 5380 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5381 struct mem_cgroup_event *event, *tmp; 5382 5383 /* 5384 * Unregister events and notify userspace. 5385 * Notify userspace about cgroup removing only after rmdir of cgroup 5386 * directory to avoid race between userspace and kernelspace. 5387 */ 5388 spin_lock_irq(&memcg->event_list_lock); 5389 list_for_each_entry_safe(event, tmp, &memcg->event_list, list) { 5390 list_del_init(&event->list); 5391 schedule_work(&event->remove); 5392 } 5393 spin_unlock_irq(&memcg->event_list_lock); 5394 5395 page_counter_set_min(&memcg->memory, 0); 5396 page_counter_set_low(&memcg->memory, 0); 5397 5398 memcg_offline_kmem(memcg); 5399 reparent_shrinker_deferred(memcg); 5400 wb_memcg_offline(memcg); 5401 5402 drain_all_stock(memcg); 5403 5404 mem_cgroup_id_put(memcg); 5405 } 5406 5407 static void mem_cgroup_css_released(struct cgroup_subsys_state *css) 5408 { 5409 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5410 5411 invalidate_reclaim_iterators(memcg); 5412 } 5413 5414 static void mem_cgroup_css_free(struct cgroup_subsys_state *css) 5415 { 5416 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5417 int __maybe_unused i; 5418 5419 #ifdef CONFIG_CGROUP_WRITEBACK 5420 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) 5421 wb_wait_for_completion(&memcg->cgwb_frn[i].done); 5422 #endif 5423 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket) 5424 static_branch_dec(&memcg_sockets_enabled_key); 5425 5426 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active) 5427 static_branch_dec(&memcg_sockets_enabled_key); 5428 5429 vmpressure_cleanup(&memcg->vmpressure); 5430 cancel_work_sync(&memcg->high_work); 5431 mem_cgroup_remove_from_trees(memcg); 5432 free_shrinker_info(memcg); 5433 mem_cgroup_free(memcg); 5434 } 5435 5436 /** 5437 * mem_cgroup_css_reset - reset the states of a mem_cgroup 5438 * @css: the target css 5439 * 5440 * Reset the states of the mem_cgroup associated with @css. This is 5441 * invoked when the userland requests disabling on the default hierarchy 5442 * but the memcg is pinned through dependency. The memcg should stop 5443 * applying policies and should revert to the vanilla state as it may be 5444 * made visible again. 5445 * 5446 * The current implementation only resets the essential configurations. 5447 * This needs to be expanded to cover all the visible parts. 5448 */ 5449 static void mem_cgroup_css_reset(struct cgroup_subsys_state *css) 5450 { 5451 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5452 5453 page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX); 5454 page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX); 5455 page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX); 5456 page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX); 5457 page_counter_set_min(&memcg->memory, 0); 5458 page_counter_set_low(&memcg->memory, 0); 5459 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX); 5460 memcg->soft_limit = PAGE_COUNTER_MAX; 5461 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX); 5462 memcg_wb_domain_size_changed(memcg); 5463 } 5464 5465 static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu) 5466 { 5467 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5468 struct mem_cgroup *parent = parent_mem_cgroup(memcg); 5469 struct memcg_vmstats_percpu *statc; 5470 long delta, v; 5471 int i, nid; 5472 5473 statc = per_cpu_ptr(memcg->vmstats_percpu, cpu); 5474 5475 for (i = 0; i < MEMCG_NR_STAT; i++) { 5476 /* 5477 * Collect the aggregated propagation counts of groups 5478 * below us. We're in a per-cpu loop here and this is 5479 * a global counter, so the first cycle will get them. 5480 */ 5481 delta = memcg->vmstats->state_pending[i]; 5482 if (delta) 5483 memcg->vmstats->state_pending[i] = 0; 5484 5485 /* Add CPU changes on this level since the last flush */ 5486 v = READ_ONCE(statc->state[i]); 5487 if (v != statc->state_prev[i]) { 5488 delta += v - statc->state_prev[i]; 5489 statc->state_prev[i] = v; 5490 } 5491 5492 if (!delta) 5493 continue; 5494 5495 /* Aggregate counts on this level and propagate upwards */ 5496 memcg->vmstats->state[i] += delta; 5497 if (parent) 5498 parent->vmstats->state_pending[i] += delta; 5499 } 5500 5501 for (i = 0; i < NR_MEMCG_EVENTS; i++) { 5502 delta = memcg->vmstats->events_pending[i]; 5503 if (delta) 5504 memcg->vmstats->events_pending[i] = 0; 5505 5506 v = READ_ONCE(statc->events[i]); 5507 if (v != statc->events_prev[i]) { 5508 delta += v - statc->events_prev[i]; 5509 statc->events_prev[i] = v; 5510 } 5511 5512 if (!delta) 5513 continue; 5514 5515 memcg->vmstats->events[i] += delta; 5516 if (parent) 5517 parent->vmstats->events_pending[i] += delta; 5518 } 5519 5520 for_each_node_state(nid, N_MEMORY) { 5521 struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid]; 5522 struct mem_cgroup_per_node *ppn = NULL; 5523 struct lruvec_stats_percpu *lstatc; 5524 5525 if (parent) 5526 ppn = parent->nodeinfo[nid]; 5527 5528 lstatc = per_cpu_ptr(pn->lruvec_stats_percpu, cpu); 5529 5530 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 5531 delta = pn->lruvec_stats.state_pending[i]; 5532 if (delta) 5533 pn->lruvec_stats.state_pending[i] = 0; 5534 5535 v = READ_ONCE(lstatc->state[i]); 5536 if (v != lstatc->state_prev[i]) { 5537 delta += v - lstatc->state_prev[i]; 5538 lstatc->state_prev[i] = v; 5539 } 5540 5541 if (!delta) 5542 continue; 5543 5544 pn->lruvec_stats.state[i] += delta; 5545 if (ppn) 5546 ppn->lruvec_stats.state_pending[i] += delta; 5547 } 5548 } 5549 } 5550 5551 #ifdef CONFIG_MMU 5552 /* Handlers for move charge at task migration. */ 5553 static int mem_cgroup_do_precharge(unsigned long count) 5554 { 5555 int ret; 5556 5557 /* Try a single bulk charge without reclaim first, kswapd may wake */ 5558 ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count); 5559 if (!ret) { 5560 mc.precharge += count; 5561 return ret; 5562 } 5563 5564 /* Try charges one by one with reclaim, but do not retry */ 5565 while (count--) { 5566 ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1); 5567 if (ret) 5568 return ret; 5569 mc.precharge++; 5570 cond_resched(); 5571 } 5572 return 0; 5573 } 5574 5575 union mc_target { 5576 struct page *page; 5577 swp_entry_t ent; 5578 }; 5579 5580 enum mc_target_type { 5581 MC_TARGET_NONE = 0, 5582 MC_TARGET_PAGE, 5583 MC_TARGET_SWAP, 5584 MC_TARGET_DEVICE, 5585 }; 5586 5587 static struct page *mc_handle_present_pte(struct vm_area_struct *vma, 5588 unsigned long addr, pte_t ptent) 5589 { 5590 struct page *page = vm_normal_page(vma, addr, ptent); 5591 5592 if (!page || !page_mapped(page)) 5593 return NULL; 5594 if (PageAnon(page)) { 5595 if (!(mc.flags & MOVE_ANON)) 5596 return NULL; 5597 } else { 5598 if (!(mc.flags & MOVE_FILE)) 5599 return NULL; 5600 } 5601 if (!get_page_unless_zero(page)) 5602 return NULL; 5603 5604 return page; 5605 } 5606 5607 #if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE) 5608 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma, 5609 pte_t ptent, swp_entry_t *entry) 5610 { 5611 struct page *page = NULL; 5612 swp_entry_t ent = pte_to_swp_entry(ptent); 5613 5614 if (!(mc.flags & MOVE_ANON)) 5615 return NULL; 5616 5617 /* 5618 * Handle device private pages that are not accessible by the CPU, but 5619 * stored as special swap entries in the page table. 5620 */ 5621 if (is_device_private_entry(ent)) { 5622 page = pfn_swap_entry_to_page(ent); 5623 if (!get_page_unless_zero(page)) 5624 return NULL; 5625 return page; 5626 } 5627 5628 if (non_swap_entry(ent)) 5629 return NULL; 5630 5631 /* 5632 * Because swap_cache_get_folio() updates some statistics counter, 5633 * we call find_get_page() with swapper_space directly. 5634 */ 5635 page = find_get_page(swap_address_space(ent), swp_offset(ent)); 5636 entry->val = ent.val; 5637 5638 return page; 5639 } 5640 #else 5641 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma, 5642 pte_t ptent, swp_entry_t *entry) 5643 { 5644 return NULL; 5645 } 5646 #endif 5647 5648 static struct page *mc_handle_file_pte(struct vm_area_struct *vma, 5649 unsigned long addr, pte_t ptent) 5650 { 5651 if (!vma->vm_file) /* anonymous vma */ 5652 return NULL; 5653 if (!(mc.flags & MOVE_FILE)) 5654 return NULL; 5655 5656 /* page is moved even if it's not RSS of this task(page-faulted). */ 5657 /* shmem/tmpfs may report page out on swap: account for that too. */ 5658 return find_get_incore_page(vma->vm_file->f_mapping, 5659 linear_page_index(vma, addr)); 5660 } 5661 5662 /** 5663 * mem_cgroup_move_account - move account of the page 5664 * @page: the page 5665 * @compound: charge the page as compound or small page 5666 * @from: mem_cgroup which the page is moved from. 5667 * @to: mem_cgroup which the page is moved to. @from != @to. 5668 * 5669 * The caller must make sure the page is not on LRU (isolate_page() is useful.) 5670 * 5671 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge" 5672 * from old cgroup. 5673 */ 5674 static int mem_cgroup_move_account(struct page *page, 5675 bool compound, 5676 struct mem_cgroup *from, 5677 struct mem_cgroup *to) 5678 { 5679 struct folio *folio = page_folio(page); 5680 struct lruvec *from_vec, *to_vec; 5681 struct pglist_data *pgdat; 5682 unsigned int nr_pages = compound ? folio_nr_pages(folio) : 1; 5683 int nid, ret; 5684 5685 VM_BUG_ON(from == to); 5686 VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); 5687 VM_BUG_ON(compound && !folio_test_large(folio)); 5688 5689 /* 5690 * Prevent mem_cgroup_migrate() from looking at 5691 * page's memory cgroup of its source page while we change it. 5692 */ 5693 ret = -EBUSY; 5694 if (!folio_trylock(folio)) 5695 goto out; 5696 5697 ret = -EINVAL; 5698 if (folio_memcg(folio) != from) 5699 goto out_unlock; 5700 5701 pgdat = folio_pgdat(folio); 5702 from_vec = mem_cgroup_lruvec(from, pgdat); 5703 to_vec = mem_cgroup_lruvec(to, pgdat); 5704 5705 folio_memcg_lock(folio); 5706 5707 if (folio_test_anon(folio)) { 5708 if (folio_mapped(folio)) { 5709 __mod_lruvec_state(from_vec, NR_ANON_MAPPED, -nr_pages); 5710 __mod_lruvec_state(to_vec, NR_ANON_MAPPED, nr_pages); 5711 if (folio_test_transhuge(folio)) { 5712 __mod_lruvec_state(from_vec, NR_ANON_THPS, 5713 -nr_pages); 5714 __mod_lruvec_state(to_vec, NR_ANON_THPS, 5715 nr_pages); 5716 } 5717 } 5718 } else { 5719 __mod_lruvec_state(from_vec, NR_FILE_PAGES, -nr_pages); 5720 __mod_lruvec_state(to_vec, NR_FILE_PAGES, nr_pages); 5721 5722 if (folio_test_swapbacked(folio)) { 5723 __mod_lruvec_state(from_vec, NR_SHMEM, -nr_pages); 5724 __mod_lruvec_state(to_vec, NR_SHMEM, nr_pages); 5725 } 5726 5727 if (folio_mapped(folio)) { 5728 __mod_lruvec_state(from_vec, NR_FILE_MAPPED, -nr_pages); 5729 __mod_lruvec_state(to_vec, NR_FILE_MAPPED, nr_pages); 5730 } 5731 5732 if (folio_test_dirty(folio)) { 5733 struct address_space *mapping = folio_mapping(folio); 5734 5735 if (mapping_can_writeback(mapping)) { 5736 __mod_lruvec_state(from_vec, NR_FILE_DIRTY, 5737 -nr_pages); 5738 __mod_lruvec_state(to_vec, NR_FILE_DIRTY, 5739 nr_pages); 5740 } 5741 } 5742 } 5743 5744 if (folio_test_writeback(folio)) { 5745 __mod_lruvec_state(from_vec, NR_WRITEBACK, -nr_pages); 5746 __mod_lruvec_state(to_vec, NR_WRITEBACK, nr_pages); 5747 } 5748 5749 /* 5750 * All state has been migrated, let's switch to the new memcg. 5751 * 5752 * It is safe to change page's memcg here because the page 5753 * is referenced, charged, isolated, and locked: we can't race 5754 * with (un)charging, migration, LRU putback, or anything else 5755 * that would rely on a stable page's memory cgroup. 5756 * 5757 * Note that lock_page_memcg is a memcg lock, not a page lock, 5758 * to save space. As soon as we switch page's memory cgroup to a 5759 * new memcg that isn't locked, the above state can change 5760 * concurrently again. Make sure we're truly done with it. 5761 */ 5762 smp_mb(); 5763 5764 css_get(&to->css); 5765 css_put(&from->css); 5766 5767 folio->memcg_data = (unsigned long)to; 5768 5769 __folio_memcg_unlock(from); 5770 5771 ret = 0; 5772 nid = folio_nid(folio); 5773 5774 local_irq_disable(); 5775 mem_cgroup_charge_statistics(to, nr_pages); 5776 memcg_check_events(to, nid); 5777 mem_cgroup_charge_statistics(from, -nr_pages); 5778 memcg_check_events(from, nid); 5779 local_irq_enable(); 5780 out_unlock: 5781 folio_unlock(folio); 5782 out: 5783 return ret; 5784 } 5785 5786 /** 5787 * get_mctgt_type - get target type of moving charge 5788 * @vma: the vma the pte to be checked belongs 5789 * @addr: the address corresponding to the pte to be checked 5790 * @ptent: the pte to be checked 5791 * @target: the pointer the target page or swap ent will be stored(can be NULL) 5792 * 5793 * Returns 5794 * 0(MC_TARGET_NONE): if the pte is not a target for move charge. 5795 * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for 5796 * move charge. if @target is not NULL, the page is stored in target->page 5797 * with extra refcnt got(Callers should handle it). 5798 * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a 5799 * target for charge migration. if @target is not NULL, the entry is stored 5800 * in target->ent. 5801 * 3(MC_TARGET_DEVICE): like MC_TARGET_PAGE but page is device memory and 5802 * thus not on the lru. 5803 * For now we such page is charge like a regular page would be as for all 5804 * intent and purposes it is just special memory taking the place of a 5805 * regular page. 5806 * 5807 * See Documentations/vm/hmm.txt and include/linux/hmm.h 5808 * 5809 * Called with pte lock held. 5810 */ 5811 5812 static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma, 5813 unsigned long addr, pte_t ptent, union mc_target *target) 5814 { 5815 struct page *page = NULL; 5816 enum mc_target_type ret = MC_TARGET_NONE; 5817 swp_entry_t ent = { .val = 0 }; 5818 5819 if (pte_present(ptent)) 5820 page = mc_handle_present_pte(vma, addr, ptent); 5821 else if (pte_none_mostly(ptent)) 5822 /* 5823 * PTE markers should be treated as a none pte here, separated 5824 * from other swap handling below. 5825 */ 5826 page = mc_handle_file_pte(vma, addr, ptent); 5827 else if (is_swap_pte(ptent)) 5828 page = mc_handle_swap_pte(vma, ptent, &ent); 5829 5830 if (!page && !ent.val) 5831 return ret; 5832 if (page) { 5833 /* 5834 * Do only loose check w/o serialization. 5835 * mem_cgroup_move_account() checks the page is valid or 5836 * not under LRU exclusion. 5837 */ 5838 if (page_memcg(page) == mc.from) { 5839 ret = MC_TARGET_PAGE; 5840 if (is_device_private_page(page) || 5841 is_device_coherent_page(page)) 5842 ret = MC_TARGET_DEVICE; 5843 if (target) 5844 target->page = page; 5845 } 5846 if (!ret || !target) 5847 put_page(page); 5848 } 5849 /* 5850 * There is a swap entry and a page doesn't exist or isn't charged. 5851 * But we cannot move a tail-page in a THP. 5852 */ 5853 if (ent.val && !ret && (!page || !PageTransCompound(page)) && 5854 mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) { 5855 ret = MC_TARGET_SWAP; 5856 if (target) 5857 target->ent = ent; 5858 } 5859 return ret; 5860 } 5861 5862 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5863 /* 5864 * We don't consider PMD mapped swapping or file mapped pages because THP does 5865 * not support them for now. 5866 * Caller should make sure that pmd_trans_huge(pmd) is true. 5867 */ 5868 static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma, 5869 unsigned long addr, pmd_t pmd, union mc_target *target) 5870 { 5871 struct page *page = NULL; 5872 enum mc_target_type ret = MC_TARGET_NONE; 5873 5874 if (unlikely(is_swap_pmd(pmd))) { 5875 VM_BUG_ON(thp_migration_supported() && 5876 !is_pmd_migration_entry(pmd)); 5877 return ret; 5878 } 5879 page = pmd_page(pmd); 5880 VM_BUG_ON_PAGE(!page || !PageHead(page), page); 5881 if (!(mc.flags & MOVE_ANON)) 5882 return ret; 5883 if (page_memcg(page) == mc.from) { 5884 ret = MC_TARGET_PAGE; 5885 if (target) { 5886 get_page(page); 5887 target->page = page; 5888 } 5889 } 5890 return ret; 5891 } 5892 #else 5893 static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma, 5894 unsigned long addr, pmd_t pmd, union mc_target *target) 5895 { 5896 return MC_TARGET_NONE; 5897 } 5898 #endif 5899 5900 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd, 5901 unsigned long addr, unsigned long end, 5902 struct mm_walk *walk) 5903 { 5904 struct vm_area_struct *vma = walk->vma; 5905 pte_t *pte; 5906 spinlock_t *ptl; 5907 5908 ptl = pmd_trans_huge_lock(pmd, vma); 5909 if (ptl) { 5910 /* 5911 * Note their can not be MC_TARGET_DEVICE for now as we do not 5912 * support transparent huge page with MEMORY_DEVICE_PRIVATE but 5913 * this might change. 5914 */ 5915 if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE) 5916 mc.precharge += HPAGE_PMD_NR; 5917 spin_unlock(ptl); 5918 return 0; 5919 } 5920 5921 if (pmd_trans_unstable(pmd)) 5922 return 0; 5923 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 5924 for (; addr != end; pte++, addr += PAGE_SIZE) 5925 if (get_mctgt_type(vma, addr, *pte, NULL)) 5926 mc.precharge++; /* increment precharge temporarily */ 5927 pte_unmap_unlock(pte - 1, ptl); 5928 cond_resched(); 5929 5930 return 0; 5931 } 5932 5933 static const struct mm_walk_ops precharge_walk_ops = { 5934 .pmd_entry = mem_cgroup_count_precharge_pte_range, 5935 }; 5936 5937 static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm) 5938 { 5939 unsigned long precharge; 5940 5941 mmap_read_lock(mm); 5942 walk_page_range(mm, 0, ULONG_MAX, &precharge_walk_ops, NULL); 5943 mmap_read_unlock(mm); 5944 5945 precharge = mc.precharge; 5946 mc.precharge = 0; 5947 5948 return precharge; 5949 } 5950 5951 static int mem_cgroup_precharge_mc(struct mm_struct *mm) 5952 { 5953 unsigned long precharge = mem_cgroup_count_precharge(mm); 5954 5955 VM_BUG_ON(mc.moving_task); 5956 mc.moving_task = current; 5957 return mem_cgroup_do_precharge(precharge); 5958 } 5959 5960 /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */ 5961 static void __mem_cgroup_clear_mc(void) 5962 { 5963 struct mem_cgroup *from = mc.from; 5964 struct mem_cgroup *to = mc.to; 5965 5966 /* we must uncharge all the leftover precharges from mc.to */ 5967 if (mc.precharge) { 5968 cancel_charge(mc.to, mc.precharge); 5969 mc.precharge = 0; 5970 } 5971 /* 5972 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so 5973 * we must uncharge here. 5974 */ 5975 if (mc.moved_charge) { 5976 cancel_charge(mc.from, mc.moved_charge); 5977 mc.moved_charge = 0; 5978 } 5979 /* we must fixup refcnts and charges */ 5980 if (mc.moved_swap) { 5981 /* uncharge swap account from the old cgroup */ 5982 if (!mem_cgroup_is_root(mc.from)) 5983 page_counter_uncharge(&mc.from->memsw, mc.moved_swap); 5984 5985 mem_cgroup_id_put_many(mc.from, mc.moved_swap); 5986 5987 /* 5988 * we charged both to->memory and to->memsw, so we 5989 * should uncharge to->memory. 5990 */ 5991 if (!mem_cgroup_is_root(mc.to)) 5992 page_counter_uncharge(&mc.to->memory, mc.moved_swap); 5993 5994 mc.moved_swap = 0; 5995 } 5996 memcg_oom_recover(from); 5997 memcg_oom_recover(to); 5998 wake_up_all(&mc.waitq); 5999 } 6000 6001 static void mem_cgroup_clear_mc(void) 6002 { 6003 struct mm_struct *mm = mc.mm; 6004 6005 /* 6006 * we must clear moving_task before waking up waiters at the end of 6007 * task migration. 6008 */ 6009 mc.moving_task = NULL; 6010 __mem_cgroup_clear_mc(); 6011 spin_lock(&mc.lock); 6012 mc.from = NULL; 6013 mc.to = NULL; 6014 mc.mm = NULL; 6015 spin_unlock(&mc.lock); 6016 6017 mmput(mm); 6018 } 6019 6020 static int mem_cgroup_can_attach(struct cgroup_taskset *tset) 6021 { 6022 struct cgroup_subsys_state *css; 6023 struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */ 6024 struct mem_cgroup *from; 6025 struct task_struct *leader, *p; 6026 struct mm_struct *mm; 6027 unsigned long move_flags; 6028 int ret = 0; 6029 6030 /* charge immigration isn't supported on the default hierarchy */ 6031 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 6032 return 0; 6033 6034 /* 6035 * Multi-process migrations only happen on the default hierarchy 6036 * where charge immigration is not used. Perform charge 6037 * immigration if @tset contains a leader and whine if there are 6038 * multiple. 6039 */ 6040 p = NULL; 6041 cgroup_taskset_for_each_leader(leader, css, tset) { 6042 WARN_ON_ONCE(p); 6043 p = leader; 6044 memcg = mem_cgroup_from_css(css); 6045 } 6046 if (!p) 6047 return 0; 6048 6049 /* 6050 * We are now committed to this value whatever it is. Changes in this 6051 * tunable will only affect upcoming migrations, not the current one. 6052 * So we need to save it, and keep it going. 6053 */ 6054 move_flags = READ_ONCE(memcg->move_charge_at_immigrate); 6055 if (!move_flags) 6056 return 0; 6057 6058 from = mem_cgroup_from_task(p); 6059 6060 VM_BUG_ON(from == memcg); 6061 6062 mm = get_task_mm(p); 6063 if (!mm) 6064 return 0; 6065 /* We move charges only when we move a owner of the mm */ 6066 if (mm->owner == p) { 6067 VM_BUG_ON(mc.from); 6068 VM_BUG_ON(mc.to); 6069 VM_BUG_ON(mc.precharge); 6070 VM_BUG_ON(mc.moved_charge); 6071 VM_BUG_ON(mc.moved_swap); 6072 6073 spin_lock(&mc.lock); 6074 mc.mm = mm; 6075 mc.from = from; 6076 mc.to = memcg; 6077 mc.flags = move_flags; 6078 spin_unlock(&mc.lock); 6079 /* We set mc.moving_task later */ 6080 6081 ret = mem_cgroup_precharge_mc(mm); 6082 if (ret) 6083 mem_cgroup_clear_mc(); 6084 } else { 6085 mmput(mm); 6086 } 6087 return ret; 6088 } 6089 6090 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset) 6091 { 6092 if (mc.to) 6093 mem_cgroup_clear_mc(); 6094 } 6095 6096 static int mem_cgroup_move_charge_pte_range(pmd_t *pmd, 6097 unsigned long addr, unsigned long end, 6098 struct mm_walk *walk) 6099 { 6100 int ret = 0; 6101 struct vm_area_struct *vma = walk->vma; 6102 pte_t *pte; 6103 spinlock_t *ptl; 6104 enum mc_target_type target_type; 6105 union mc_target target; 6106 struct page *page; 6107 6108 ptl = pmd_trans_huge_lock(pmd, vma); 6109 if (ptl) { 6110 if (mc.precharge < HPAGE_PMD_NR) { 6111 spin_unlock(ptl); 6112 return 0; 6113 } 6114 target_type = get_mctgt_type_thp(vma, addr, *pmd, &target); 6115 if (target_type == MC_TARGET_PAGE) { 6116 page = target.page; 6117 if (!isolate_lru_page(page)) { 6118 if (!mem_cgroup_move_account(page, true, 6119 mc.from, mc.to)) { 6120 mc.precharge -= HPAGE_PMD_NR; 6121 mc.moved_charge += HPAGE_PMD_NR; 6122 } 6123 putback_lru_page(page); 6124 } 6125 put_page(page); 6126 } else if (target_type == MC_TARGET_DEVICE) { 6127 page = target.page; 6128 if (!mem_cgroup_move_account(page, true, 6129 mc.from, mc.to)) { 6130 mc.precharge -= HPAGE_PMD_NR; 6131 mc.moved_charge += HPAGE_PMD_NR; 6132 } 6133 put_page(page); 6134 } 6135 spin_unlock(ptl); 6136 return 0; 6137 } 6138 6139 if (pmd_trans_unstable(pmd)) 6140 return 0; 6141 retry: 6142 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 6143 for (; addr != end; addr += PAGE_SIZE) { 6144 pte_t ptent = *(pte++); 6145 bool device = false; 6146 swp_entry_t ent; 6147 6148 if (!mc.precharge) 6149 break; 6150 6151 switch (get_mctgt_type(vma, addr, ptent, &target)) { 6152 case MC_TARGET_DEVICE: 6153 device = true; 6154 fallthrough; 6155 case MC_TARGET_PAGE: 6156 page = target.page; 6157 /* 6158 * We can have a part of the split pmd here. Moving it 6159 * can be done but it would be too convoluted so simply 6160 * ignore such a partial THP and keep it in original 6161 * memcg. There should be somebody mapping the head. 6162 */ 6163 if (PageTransCompound(page)) 6164 goto put; 6165 if (!device && isolate_lru_page(page)) 6166 goto put; 6167 if (!mem_cgroup_move_account(page, false, 6168 mc.from, mc.to)) { 6169 mc.precharge--; 6170 /* we uncharge from mc.from later. */ 6171 mc.moved_charge++; 6172 } 6173 if (!device) 6174 putback_lru_page(page); 6175 put: /* get_mctgt_type() gets the page */ 6176 put_page(page); 6177 break; 6178 case MC_TARGET_SWAP: 6179 ent = target.ent; 6180 if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) { 6181 mc.precharge--; 6182 mem_cgroup_id_get_many(mc.to, 1); 6183 /* we fixup other refcnts and charges later. */ 6184 mc.moved_swap++; 6185 } 6186 break; 6187 default: 6188 break; 6189 } 6190 } 6191 pte_unmap_unlock(pte - 1, ptl); 6192 cond_resched(); 6193 6194 if (addr != end) { 6195 /* 6196 * We have consumed all precharges we got in can_attach(). 6197 * We try charge one by one, but don't do any additional 6198 * charges to mc.to if we have failed in charge once in attach() 6199 * phase. 6200 */ 6201 ret = mem_cgroup_do_precharge(1); 6202 if (!ret) 6203 goto retry; 6204 } 6205 6206 return ret; 6207 } 6208 6209 static const struct mm_walk_ops charge_walk_ops = { 6210 .pmd_entry = mem_cgroup_move_charge_pte_range, 6211 }; 6212 6213 static void mem_cgroup_move_charge(void) 6214 { 6215 lru_add_drain_all(); 6216 /* 6217 * Signal lock_page_memcg() to take the memcg's move_lock 6218 * while we're moving its pages to another memcg. Then wait 6219 * for already started RCU-only updates to finish. 6220 */ 6221 atomic_inc(&mc.from->moving_account); 6222 synchronize_rcu(); 6223 retry: 6224 if (unlikely(!mmap_read_trylock(mc.mm))) { 6225 /* 6226 * Someone who are holding the mmap_lock might be waiting in 6227 * waitq. So we cancel all extra charges, wake up all waiters, 6228 * and retry. Because we cancel precharges, we might not be able 6229 * to move enough charges, but moving charge is a best-effort 6230 * feature anyway, so it wouldn't be a big problem. 6231 */ 6232 __mem_cgroup_clear_mc(); 6233 cond_resched(); 6234 goto retry; 6235 } 6236 /* 6237 * When we have consumed all precharges and failed in doing 6238 * additional charge, the page walk just aborts. 6239 */ 6240 walk_page_range(mc.mm, 0, ULONG_MAX, &charge_walk_ops, NULL); 6241 mmap_read_unlock(mc.mm); 6242 atomic_dec(&mc.from->moving_account); 6243 } 6244 6245 static void mem_cgroup_move_task(void) 6246 { 6247 if (mc.to) { 6248 mem_cgroup_move_charge(); 6249 mem_cgroup_clear_mc(); 6250 } 6251 } 6252 #else /* !CONFIG_MMU */ 6253 static int mem_cgroup_can_attach(struct cgroup_taskset *tset) 6254 { 6255 return 0; 6256 } 6257 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset) 6258 { 6259 } 6260 static void mem_cgroup_move_task(void) 6261 { 6262 } 6263 #endif 6264 6265 #ifdef CONFIG_LRU_GEN 6266 static void mem_cgroup_attach(struct cgroup_taskset *tset) 6267 { 6268 struct task_struct *task; 6269 struct cgroup_subsys_state *css; 6270 6271 /* find the first leader if there is any */ 6272 cgroup_taskset_for_each_leader(task, css, tset) 6273 break; 6274 6275 if (!task) 6276 return; 6277 6278 task_lock(task); 6279 if (task->mm && READ_ONCE(task->mm->owner) == task) 6280 lru_gen_migrate_mm(task->mm); 6281 task_unlock(task); 6282 } 6283 #else 6284 static void mem_cgroup_attach(struct cgroup_taskset *tset) 6285 { 6286 } 6287 #endif /* CONFIG_LRU_GEN */ 6288 6289 static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value) 6290 { 6291 if (value == PAGE_COUNTER_MAX) 6292 seq_puts(m, "max\n"); 6293 else 6294 seq_printf(m, "%llu\n", (u64)value * PAGE_SIZE); 6295 6296 return 0; 6297 } 6298 6299 static u64 memory_current_read(struct cgroup_subsys_state *css, 6300 struct cftype *cft) 6301 { 6302 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 6303 6304 return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE; 6305 } 6306 6307 static u64 memory_peak_read(struct cgroup_subsys_state *css, 6308 struct cftype *cft) 6309 { 6310 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 6311 6312 return (u64)memcg->memory.watermark * PAGE_SIZE; 6313 } 6314 6315 static int memory_min_show(struct seq_file *m, void *v) 6316 { 6317 return seq_puts_memcg_tunable(m, 6318 READ_ONCE(mem_cgroup_from_seq(m)->memory.min)); 6319 } 6320 6321 static ssize_t memory_min_write(struct kernfs_open_file *of, 6322 char *buf, size_t nbytes, loff_t off) 6323 { 6324 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 6325 unsigned long min; 6326 int err; 6327 6328 buf = strstrip(buf); 6329 err = page_counter_memparse(buf, "max", &min); 6330 if (err) 6331 return err; 6332 6333 page_counter_set_min(&memcg->memory, min); 6334 6335 return nbytes; 6336 } 6337 6338 static int memory_low_show(struct seq_file *m, void *v) 6339 { 6340 return seq_puts_memcg_tunable(m, 6341 READ_ONCE(mem_cgroup_from_seq(m)->memory.low)); 6342 } 6343 6344 static ssize_t memory_low_write(struct kernfs_open_file *of, 6345 char *buf, size_t nbytes, loff_t off) 6346 { 6347 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 6348 unsigned long low; 6349 int err; 6350 6351 buf = strstrip(buf); 6352 err = page_counter_memparse(buf, "max", &low); 6353 if (err) 6354 return err; 6355 6356 page_counter_set_low(&memcg->memory, low); 6357 6358 return nbytes; 6359 } 6360 6361 static int memory_high_show(struct seq_file *m, void *v) 6362 { 6363 return seq_puts_memcg_tunable(m, 6364 READ_ONCE(mem_cgroup_from_seq(m)->memory.high)); 6365 } 6366 6367 static ssize_t memory_high_write(struct kernfs_open_file *of, 6368 char *buf, size_t nbytes, loff_t off) 6369 { 6370 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 6371 unsigned int nr_retries = MAX_RECLAIM_RETRIES; 6372 bool drained = false; 6373 unsigned long high; 6374 int err; 6375 6376 buf = strstrip(buf); 6377 err = page_counter_memparse(buf, "max", &high); 6378 if (err) 6379 return err; 6380 6381 page_counter_set_high(&memcg->memory, high); 6382 6383 for (;;) { 6384 unsigned long nr_pages = page_counter_read(&memcg->memory); 6385 unsigned long reclaimed; 6386 6387 if (nr_pages <= high) 6388 break; 6389 6390 if (signal_pending(current)) 6391 break; 6392 6393 if (!drained) { 6394 drain_all_stock(memcg); 6395 drained = true; 6396 continue; 6397 } 6398 6399 reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high, 6400 GFP_KERNEL, MEMCG_RECLAIM_MAY_SWAP); 6401 6402 if (!reclaimed && !nr_retries--) 6403 break; 6404 } 6405 6406 memcg_wb_domain_size_changed(memcg); 6407 return nbytes; 6408 } 6409 6410 static int memory_max_show(struct seq_file *m, void *v) 6411 { 6412 return seq_puts_memcg_tunable(m, 6413 READ_ONCE(mem_cgroup_from_seq(m)->memory.max)); 6414 } 6415 6416 static ssize_t memory_max_write(struct kernfs_open_file *of, 6417 char *buf, size_t nbytes, loff_t off) 6418 { 6419 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 6420 unsigned int nr_reclaims = MAX_RECLAIM_RETRIES; 6421 bool drained = false; 6422 unsigned long max; 6423 int err; 6424 6425 buf = strstrip(buf); 6426 err = page_counter_memparse(buf, "max", &max); 6427 if (err) 6428 return err; 6429 6430 xchg(&memcg->memory.max, max); 6431 6432 for (;;) { 6433 unsigned long nr_pages = page_counter_read(&memcg->memory); 6434 6435 if (nr_pages <= max) 6436 break; 6437 6438 if (signal_pending(current)) 6439 break; 6440 6441 if (!drained) { 6442 drain_all_stock(memcg); 6443 drained = true; 6444 continue; 6445 } 6446 6447 if (nr_reclaims) { 6448 if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max, 6449 GFP_KERNEL, MEMCG_RECLAIM_MAY_SWAP)) 6450 nr_reclaims--; 6451 continue; 6452 } 6453 6454 memcg_memory_event(memcg, MEMCG_OOM); 6455 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0)) 6456 break; 6457 } 6458 6459 memcg_wb_domain_size_changed(memcg); 6460 return nbytes; 6461 } 6462 6463 static void __memory_events_show(struct seq_file *m, atomic_long_t *events) 6464 { 6465 seq_printf(m, "low %lu\n", atomic_long_read(&events[MEMCG_LOW])); 6466 seq_printf(m, "high %lu\n", atomic_long_read(&events[MEMCG_HIGH])); 6467 seq_printf(m, "max %lu\n", atomic_long_read(&events[MEMCG_MAX])); 6468 seq_printf(m, "oom %lu\n", atomic_long_read(&events[MEMCG_OOM])); 6469 seq_printf(m, "oom_kill %lu\n", 6470 atomic_long_read(&events[MEMCG_OOM_KILL])); 6471 seq_printf(m, "oom_group_kill %lu\n", 6472 atomic_long_read(&events[MEMCG_OOM_GROUP_KILL])); 6473 } 6474 6475 static int memory_events_show(struct seq_file *m, void *v) 6476 { 6477 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 6478 6479 __memory_events_show(m, memcg->memory_events); 6480 return 0; 6481 } 6482 6483 static int memory_events_local_show(struct seq_file *m, void *v) 6484 { 6485 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 6486 6487 __memory_events_show(m, memcg->memory_events_local); 6488 return 0; 6489 } 6490 6491 static int memory_stat_show(struct seq_file *m, void *v) 6492 { 6493 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 6494 char *buf = kmalloc(PAGE_SIZE, GFP_KERNEL); 6495 6496 if (!buf) 6497 return -ENOMEM; 6498 memory_stat_format(memcg, buf, PAGE_SIZE); 6499 seq_puts(m, buf); 6500 kfree(buf); 6501 return 0; 6502 } 6503 6504 #ifdef CONFIG_NUMA 6505 static inline unsigned long lruvec_page_state_output(struct lruvec *lruvec, 6506 int item) 6507 { 6508 return lruvec_page_state(lruvec, item) * memcg_page_state_unit(item); 6509 } 6510 6511 static int memory_numa_stat_show(struct seq_file *m, void *v) 6512 { 6513 int i; 6514 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 6515 6516 mem_cgroup_flush_stats(); 6517 6518 for (i = 0; i < ARRAY_SIZE(memory_stats); i++) { 6519 int nid; 6520 6521 if (memory_stats[i].idx >= NR_VM_NODE_STAT_ITEMS) 6522 continue; 6523 6524 seq_printf(m, "%s", memory_stats[i].name); 6525 for_each_node_state(nid, N_MEMORY) { 6526 u64 size; 6527 struct lruvec *lruvec; 6528 6529 lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid)); 6530 size = lruvec_page_state_output(lruvec, 6531 memory_stats[i].idx); 6532 seq_printf(m, " N%d=%llu", nid, size); 6533 } 6534 seq_putc(m, '\n'); 6535 } 6536 6537 return 0; 6538 } 6539 #endif 6540 6541 static int memory_oom_group_show(struct seq_file *m, void *v) 6542 { 6543 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 6544 6545 seq_printf(m, "%d\n", memcg->oom_group); 6546 6547 return 0; 6548 } 6549 6550 static ssize_t memory_oom_group_write(struct kernfs_open_file *of, 6551 char *buf, size_t nbytes, loff_t off) 6552 { 6553 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 6554 int ret, oom_group; 6555 6556 buf = strstrip(buf); 6557 if (!buf) 6558 return -EINVAL; 6559 6560 ret = kstrtoint(buf, 0, &oom_group); 6561 if (ret) 6562 return ret; 6563 6564 if (oom_group != 0 && oom_group != 1) 6565 return -EINVAL; 6566 6567 memcg->oom_group = oom_group; 6568 6569 return nbytes; 6570 } 6571 6572 static ssize_t memory_reclaim(struct kernfs_open_file *of, char *buf, 6573 size_t nbytes, loff_t off) 6574 { 6575 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 6576 unsigned int nr_retries = MAX_RECLAIM_RETRIES; 6577 unsigned long nr_to_reclaim, nr_reclaimed = 0; 6578 unsigned int reclaim_options; 6579 int err; 6580 6581 buf = strstrip(buf); 6582 err = page_counter_memparse(buf, "", &nr_to_reclaim); 6583 if (err) 6584 return err; 6585 6586 reclaim_options = MEMCG_RECLAIM_MAY_SWAP | MEMCG_RECLAIM_PROACTIVE; 6587 while (nr_reclaimed < nr_to_reclaim) { 6588 unsigned long reclaimed; 6589 6590 if (signal_pending(current)) 6591 return -EINTR; 6592 6593 /* 6594 * This is the final attempt, drain percpu lru caches in the 6595 * hope of introducing more evictable pages for 6596 * try_to_free_mem_cgroup_pages(). 6597 */ 6598 if (!nr_retries) 6599 lru_add_drain_all(); 6600 6601 reclaimed = try_to_free_mem_cgroup_pages(memcg, 6602 nr_to_reclaim - nr_reclaimed, 6603 GFP_KERNEL, reclaim_options); 6604 6605 if (!reclaimed && !nr_retries--) 6606 return -EAGAIN; 6607 6608 nr_reclaimed += reclaimed; 6609 } 6610 6611 return nbytes; 6612 } 6613 6614 static struct cftype memory_files[] = { 6615 { 6616 .name = "current", 6617 .flags = CFTYPE_NOT_ON_ROOT, 6618 .read_u64 = memory_current_read, 6619 }, 6620 { 6621 .name = "peak", 6622 .flags = CFTYPE_NOT_ON_ROOT, 6623 .read_u64 = memory_peak_read, 6624 }, 6625 { 6626 .name = "min", 6627 .flags = CFTYPE_NOT_ON_ROOT, 6628 .seq_show = memory_min_show, 6629 .write = memory_min_write, 6630 }, 6631 { 6632 .name = "low", 6633 .flags = CFTYPE_NOT_ON_ROOT, 6634 .seq_show = memory_low_show, 6635 .write = memory_low_write, 6636 }, 6637 { 6638 .name = "high", 6639 .flags = CFTYPE_NOT_ON_ROOT, 6640 .seq_show = memory_high_show, 6641 .write = memory_high_write, 6642 }, 6643 { 6644 .name = "max", 6645 .flags = CFTYPE_NOT_ON_ROOT, 6646 .seq_show = memory_max_show, 6647 .write = memory_max_write, 6648 }, 6649 { 6650 .name = "events", 6651 .flags = CFTYPE_NOT_ON_ROOT, 6652 .file_offset = offsetof(struct mem_cgroup, events_file), 6653 .seq_show = memory_events_show, 6654 }, 6655 { 6656 .name = "events.local", 6657 .flags = CFTYPE_NOT_ON_ROOT, 6658 .file_offset = offsetof(struct mem_cgroup, events_local_file), 6659 .seq_show = memory_events_local_show, 6660 }, 6661 { 6662 .name = "stat", 6663 .seq_show = memory_stat_show, 6664 }, 6665 #ifdef CONFIG_NUMA 6666 { 6667 .name = "numa_stat", 6668 .seq_show = memory_numa_stat_show, 6669 }, 6670 #endif 6671 { 6672 .name = "oom.group", 6673 .flags = CFTYPE_NOT_ON_ROOT | CFTYPE_NS_DELEGATABLE, 6674 .seq_show = memory_oom_group_show, 6675 .write = memory_oom_group_write, 6676 }, 6677 { 6678 .name = "reclaim", 6679 .flags = CFTYPE_NS_DELEGATABLE, 6680 .write = memory_reclaim, 6681 }, 6682 { } /* terminate */ 6683 }; 6684 6685 struct cgroup_subsys memory_cgrp_subsys = { 6686 .css_alloc = mem_cgroup_css_alloc, 6687 .css_online = mem_cgroup_css_online, 6688 .css_offline = mem_cgroup_css_offline, 6689 .css_released = mem_cgroup_css_released, 6690 .css_free = mem_cgroup_css_free, 6691 .css_reset = mem_cgroup_css_reset, 6692 .css_rstat_flush = mem_cgroup_css_rstat_flush, 6693 .can_attach = mem_cgroup_can_attach, 6694 .attach = mem_cgroup_attach, 6695 .cancel_attach = mem_cgroup_cancel_attach, 6696 .post_attach = mem_cgroup_move_task, 6697 .dfl_cftypes = memory_files, 6698 .legacy_cftypes = mem_cgroup_legacy_files, 6699 .early_init = 0, 6700 }; 6701 6702 /* 6703 * This function calculates an individual cgroup's effective 6704 * protection which is derived from its own memory.min/low, its 6705 * parent's and siblings' settings, as well as the actual memory 6706 * distribution in the tree. 6707 * 6708 * The following rules apply to the effective protection values: 6709 * 6710 * 1. At the first level of reclaim, effective protection is equal to 6711 * the declared protection in memory.min and memory.low. 6712 * 6713 * 2. To enable safe delegation of the protection configuration, at 6714 * subsequent levels the effective protection is capped to the 6715 * parent's effective protection. 6716 * 6717 * 3. To make complex and dynamic subtrees easier to configure, the 6718 * user is allowed to overcommit the declared protection at a given 6719 * level. If that is the case, the parent's effective protection is 6720 * distributed to the children in proportion to how much protection 6721 * they have declared and how much of it they are utilizing. 6722 * 6723 * This makes distribution proportional, but also work-conserving: 6724 * if one cgroup claims much more protection than it uses memory, 6725 * the unused remainder is available to its siblings. 6726 * 6727 * 4. Conversely, when the declared protection is undercommitted at a 6728 * given level, the distribution of the larger parental protection 6729 * budget is NOT proportional. A cgroup's protection from a sibling 6730 * is capped to its own memory.min/low setting. 6731 * 6732 * 5. However, to allow protecting recursive subtrees from each other 6733 * without having to declare each individual cgroup's fixed share 6734 * of the ancestor's claim to protection, any unutilized - 6735 * "floating" - protection from up the tree is distributed in 6736 * proportion to each cgroup's *usage*. This makes the protection 6737 * neutral wrt sibling cgroups and lets them compete freely over 6738 * the shared parental protection budget, but it protects the 6739 * subtree as a whole from neighboring subtrees. 6740 * 6741 * Note that 4. and 5. are not in conflict: 4. is about protecting 6742 * against immediate siblings whereas 5. is about protecting against 6743 * neighboring subtrees. 6744 */ 6745 static unsigned long effective_protection(unsigned long usage, 6746 unsigned long parent_usage, 6747 unsigned long setting, 6748 unsigned long parent_effective, 6749 unsigned long siblings_protected) 6750 { 6751 unsigned long protected; 6752 unsigned long ep; 6753 6754 protected = min(usage, setting); 6755 /* 6756 * If all cgroups at this level combined claim and use more 6757 * protection then what the parent affords them, distribute 6758 * shares in proportion to utilization. 6759 * 6760 * We are using actual utilization rather than the statically 6761 * claimed protection in order to be work-conserving: claimed 6762 * but unused protection is available to siblings that would 6763 * otherwise get a smaller chunk than what they claimed. 6764 */ 6765 if (siblings_protected > parent_effective) 6766 return protected * parent_effective / siblings_protected; 6767 6768 /* 6769 * Ok, utilized protection of all children is within what the 6770 * parent affords them, so we know whatever this child claims 6771 * and utilizes is effectively protected. 6772 * 6773 * If there is unprotected usage beyond this value, reclaim 6774 * will apply pressure in proportion to that amount. 6775 * 6776 * If there is unutilized protection, the cgroup will be fully 6777 * shielded from reclaim, but we do return a smaller value for 6778 * protection than what the group could enjoy in theory. This 6779 * is okay. With the overcommit distribution above, effective 6780 * protection is always dependent on how memory is actually 6781 * consumed among the siblings anyway. 6782 */ 6783 ep = protected; 6784 6785 /* 6786 * If the children aren't claiming (all of) the protection 6787 * afforded to them by the parent, distribute the remainder in 6788 * proportion to the (unprotected) memory of each cgroup. That 6789 * way, cgroups that aren't explicitly prioritized wrt each 6790 * other compete freely over the allowance, but they are 6791 * collectively protected from neighboring trees. 6792 * 6793 * We're using unprotected memory for the weight so that if 6794 * some cgroups DO claim explicit protection, we don't protect 6795 * the same bytes twice. 6796 * 6797 * Check both usage and parent_usage against the respective 6798 * protected values. One should imply the other, but they 6799 * aren't read atomically - make sure the division is sane. 6800 */ 6801 if (!(cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT)) 6802 return ep; 6803 if (parent_effective > siblings_protected && 6804 parent_usage > siblings_protected && 6805 usage > protected) { 6806 unsigned long unclaimed; 6807 6808 unclaimed = parent_effective - siblings_protected; 6809 unclaimed *= usage - protected; 6810 unclaimed /= parent_usage - siblings_protected; 6811 6812 ep += unclaimed; 6813 } 6814 6815 return ep; 6816 } 6817 6818 /** 6819 * mem_cgroup_calculate_protection - check if memory consumption is in the normal range 6820 * @root: the top ancestor of the sub-tree being checked 6821 * @memcg: the memory cgroup to check 6822 * 6823 * WARNING: This function is not stateless! It can only be used as part 6824 * of a top-down tree iteration, not for isolated queries. 6825 */ 6826 void mem_cgroup_calculate_protection(struct mem_cgroup *root, 6827 struct mem_cgroup *memcg) 6828 { 6829 unsigned long usage, parent_usage; 6830 struct mem_cgroup *parent; 6831 6832 if (mem_cgroup_disabled()) 6833 return; 6834 6835 if (!root) 6836 root = root_mem_cgroup; 6837 6838 /* 6839 * Effective values of the reclaim targets are ignored so they 6840 * can be stale. Have a look at mem_cgroup_protection for more 6841 * details. 6842 * TODO: calculation should be more robust so that we do not need 6843 * that special casing. 6844 */ 6845 if (memcg == root) 6846 return; 6847 6848 usage = page_counter_read(&memcg->memory); 6849 if (!usage) 6850 return; 6851 6852 parent = parent_mem_cgroup(memcg); 6853 6854 if (parent == root) { 6855 memcg->memory.emin = READ_ONCE(memcg->memory.min); 6856 memcg->memory.elow = READ_ONCE(memcg->memory.low); 6857 return; 6858 } 6859 6860 parent_usage = page_counter_read(&parent->memory); 6861 6862 WRITE_ONCE(memcg->memory.emin, effective_protection(usage, parent_usage, 6863 READ_ONCE(memcg->memory.min), 6864 READ_ONCE(parent->memory.emin), 6865 atomic_long_read(&parent->memory.children_min_usage))); 6866 6867 WRITE_ONCE(memcg->memory.elow, effective_protection(usage, parent_usage, 6868 READ_ONCE(memcg->memory.low), 6869 READ_ONCE(parent->memory.elow), 6870 atomic_long_read(&parent->memory.children_low_usage))); 6871 } 6872 6873 static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg, 6874 gfp_t gfp) 6875 { 6876 long nr_pages = folio_nr_pages(folio); 6877 int ret; 6878 6879 ret = try_charge(memcg, gfp, nr_pages); 6880 if (ret) 6881 goto out; 6882 6883 css_get(&memcg->css); 6884 commit_charge(folio, memcg); 6885 6886 local_irq_disable(); 6887 mem_cgroup_charge_statistics(memcg, nr_pages); 6888 memcg_check_events(memcg, folio_nid(folio)); 6889 local_irq_enable(); 6890 out: 6891 return ret; 6892 } 6893 6894 int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp) 6895 { 6896 struct mem_cgroup *memcg; 6897 int ret; 6898 6899 memcg = get_mem_cgroup_from_mm(mm); 6900 ret = charge_memcg(folio, memcg, gfp); 6901 css_put(&memcg->css); 6902 6903 return ret; 6904 } 6905 6906 /** 6907 * mem_cgroup_swapin_charge_folio - Charge a newly allocated folio for swapin. 6908 * @folio: folio to charge. 6909 * @mm: mm context of the victim 6910 * @gfp: reclaim mode 6911 * @entry: swap entry for which the folio is allocated 6912 * 6913 * This function charges a folio allocated for swapin. Please call this before 6914 * adding the folio to the swapcache. 6915 * 6916 * Returns 0 on success. Otherwise, an error code is returned. 6917 */ 6918 int mem_cgroup_swapin_charge_folio(struct folio *folio, struct mm_struct *mm, 6919 gfp_t gfp, swp_entry_t entry) 6920 { 6921 struct mem_cgroup *memcg; 6922 unsigned short id; 6923 int ret; 6924 6925 if (mem_cgroup_disabled()) 6926 return 0; 6927 6928 id = lookup_swap_cgroup_id(entry); 6929 rcu_read_lock(); 6930 memcg = mem_cgroup_from_id(id); 6931 if (!memcg || !css_tryget_online(&memcg->css)) 6932 memcg = get_mem_cgroup_from_mm(mm); 6933 rcu_read_unlock(); 6934 6935 ret = charge_memcg(folio, memcg, gfp); 6936 6937 css_put(&memcg->css); 6938 return ret; 6939 } 6940 6941 /* 6942 * mem_cgroup_swapin_uncharge_swap - uncharge swap slot 6943 * @entry: swap entry for which the page is charged 6944 * 6945 * Call this function after successfully adding the charged page to swapcache. 6946 * 6947 * Note: This function assumes the page for which swap slot is being uncharged 6948 * is order 0 page. 6949 */ 6950 void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry) 6951 { 6952 /* 6953 * Cgroup1's unified memory+swap counter has been charged with the 6954 * new swapcache page, finish the transfer by uncharging the swap 6955 * slot. The swap slot would also get uncharged when it dies, but 6956 * it can stick around indefinitely and we'd count the page twice 6957 * the entire time. 6958 * 6959 * Cgroup2 has separate resource counters for memory and swap, 6960 * so this is a non-issue here. Memory and swap charge lifetimes 6961 * correspond 1:1 to page and swap slot lifetimes: we charge the 6962 * page to memory here, and uncharge swap when the slot is freed. 6963 */ 6964 if (!mem_cgroup_disabled() && do_memsw_account()) { 6965 /* 6966 * The swap entry might not get freed for a long time, 6967 * let's not wait for it. The page already received a 6968 * memory+swap charge, drop the swap entry duplicate. 6969 */ 6970 mem_cgroup_uncharge_swap(entry, 1); 6971 } 6972 } 6973 6974 struct uncharge_gather { 6975 struct mem_cgroup *memcg; 6976 unsigned long nr_memory; 6977 unsigned long pgpgout; 6978 unsigned long nr_kmem; 6979 int nid; 6980 }; 6981 6982 static inline void uncharge_gather_clear(struct uncharge_gather *ug) 6983 { 6984 memset(ug, 0, sizeof(*ug)); 6985 } 6986 6987 static void uncharge_batch(const struct uncharge_gather *ug) 6988 { 6989 unsigned long flags; 6990 6991 if (ug->nr_memory) { 6992 page_counter_uncharge(&ug->memcg->memory, ug->nr_memory); 6993 if (do_memsw_account()) 6994 page_counter_uncharge(&ug->memcg->memsw, ug->nr_memory); 6995 if (ug->nr_kmem) 6996 memcg_account_kmem(ug->memcg, -ug->nr_kmem); 6997 memcg_oom_recover(ug->memcg); 6998 } 6999 7000 local_irq_save(flags); 7001 __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout); 7002 __this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_memory); 7003 memcg_check_events(ug->memcg, ug->nid); 7004 local_irq_restore(flags); 7005 7006 /* drop reference from uncharge_folio */ 7007 css_put(&ug->memcg->css); 7008 } 7009 7010 static void uncharge_folio(struct folio *folio, struct uncharge_gather *ug) 7011 { 7012 long nr_pages; 7013 struct mem_cgroup *memcg; 7014 struct obj_cgroup *objcg; 7015 7016 VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); 7017 7018 /* 7019 * Nobody should be changing or seriously looking at 7020 * folio memcg or objcg at this point, we have fully 7021 * exclusive access to the folio. 7022 */ 7023 if (folio_memcg_kmem(folio)) { 7024 objcg = __folio_objcg(folio); 7025 /* 7026 * This get matches the put at the end of the function and 7027 * kmem pages do not hold memcg references anymore. 7028 */ 7029 memcg = get_mem_cgroup_from_objcg(objcg); 7030 } else { 7031 memcg = __folio_memcg(folio); 7032 } 7033 7034 if (!memcg) 7035 return; 7036 7037 if (ug->memcg != memcg) { 7038 if (ug->memcg) { 7039 uncharge_batch(ug); 7040 uncharge_gather_clear(ug); 7041 } 7042 ug->memcg = memcg; 7043 ug->nid = folio_nid(folio); 7044 7045 /* pairs with css_put in uncharge_batch */ 7046 css_get(&memcg->css); 7047 } 7048 7049 nr_pages = folio_nr_pages(folio); 7050 7051 if (folio_memcg_kmem(folio)) { 7052 ug->nr_memory += nr_pages; 7053 ug->nr_kmem += nr_pages; 7054 7055 folio->memcg_data = 0; 7056 obj_cgroup_put(objcg); 7057 } else { 7058 /* LRU pages aren't accounted at the root level */ 7059 if (!mem_cgroup_is_root(memcg)) 7060 ug->nr_memory += nr_pages; 7061 ug->pgpgout++; 7062 7063 folio->memcg_data = 0; 7064 } 7065 7066 css_put(&memcg->css); 7067 } 7068 7069 void __mem_cgroup_uncharge(struct folio *folio) 7070 { 7071 struct uncharge_gather ug; 7072 7073 /* Don't touch folio->lru of any random page, pre-check: */ 7074 if (!folio_memcg(folio)) 7075 return; 7076 7077 uncharge_gather_clear(&ug); 7078 uncharge_folio(folio, &ug); 7079 uncharge_batch(&ug); 7080 } 7081 7082 /** 7083 * __mem_cgroup_uncharge_list - uncharge a list of page 7084 * @page_list: list of pages to uncharge 7085 * 7086 * Uncharge a list of pages previously charged with 7087 * __mem_cgroup_charge(). 7088 */ 7089 void __mem_cgroup_uncharge_list(struct list_head *page_list) 7090 { 7091 struct uncharge_gather ug; 7092 struct folio *folio; 7093 7094 uncharge_gather_clear(&ug); 7095 list_for_each_entry(folio, page_list, lru) 7096 uncharge_folio(folio, &ug); 7097 if (ug.memcg) 7098 uncharge_batch(&ug); 7099 } 7100 7101 /** 7102 * mem_cgroup_migrate - Charge a folio's replacement. 7103 * @old: Currently circulating folio. 7104 * @new: Replacement folio. 7105 * 7106 * Charge @new as a replacement folio for @old. @old will 7107 * be uncharged upon free. 7108 * 7109 * Both folios must be locked, @new->mapping must be set up. 7110 */ 7111 void mem_cgroup_migrate(struct folio *old, struct folio *new) 7112 { 7113 struct mem_cgroup *memcg; 7114 long nr_pages = folio_nr_pages(new); 7115 unsigned long flags; 7116 7117 VM_BUG_ON_FOLIO(!folio_test_locked(old), old); 7118 VM_BUG_ON_FOLIO(!folio_test_locked(new), new); 7119 VM_BUG_ON_FOLIO(folio_test_anon(old) != folio_test_anon(new), new); 7120 VM_BUG_ON_FOLIO(folio_nr_pages(old) != nr_pages, new); 7121 7122 if (mem_cgroup_disabled()) 7123 return; 7124 7125 /* Page cache replacement: new folio already charged? */ 7126 if (folio_memcg(new)) 7127 return; 7128 7129 memcg = folio_memcg(old); 7130 VM_WARN_ON_ONCE_FOLIO(!memcg, old); 7131 if (!memcg) 7132 return; 7133 7134 /* Force-charge the new page. The old one will be freed soon */ 7135 if (!mem_cgroup_is_root(memcg)) { 7136 page_counter_charge(&memcg->memory, nr_pages); 7137 if (do_memsw_account()) 7138 page_counter_charge(&memcg->memsw, nr_pages); 7139 } 7140 7141 css_get(&memcg->css); 7142 commit_charge(new, memcg); 7143 7144 local_irq_save(flags); 7145 mem_cgroup_charge_statistics(memcg, nr_pages); 7146 memcg_check_events(memcg, folio_nid(new)); 7147 local_irq_restore(flags); 7148 } 7149 7150 DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key); 7151 EXPORT_SYMBOL(memcg_sockets_enabled_key); 7152 7153 void mem_cgroup_sk_alloc(struct sock *sk) 7154 { 7155 struct mem_cgroup *memcg; 7156 7157 if (!mem_cgroup_sockets_enabled) 7158 return; 7159 7160 /* Do not associate the sock with unrelated interrupted task's memcg. */ 7161 if (!in_task()) 7162 return; 7163 7164 rcu_read_lock(); 7165 memcg = mem_cgroup_from_task(current); 7166 if (memcg == root_mem_cgroup) 7167 goto out; 7168 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active) 7169 goto out; 7170 if (css_tryget(&memcg->css)) 7171 sk->sk_memcg = memcg; 7172 out: 7173 rcu_read_unlock(); 7174 } 7175 7176 void mem_cgroup_sk_free(struct sock *sk) 7177 { 7178 if (sk->sk_memcg) 7179 css_put(&sk->sk_memcg->css); 7180 } 7181 7182 /** 7183 * mem_cgroup_charge_skmem - charge socket memory 7184 * @memcg: memcg to charge 7185 * @nr_pages: number of pages to charge 7186 * @gfp_mask: reclaim mode 7187 * 7188 * Charges @nr_pages to @memcg. Returns %true if the charge fit within 7189 * @memcg's configured limit, %false if it doesn't. 7190 */ 7191 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages, 7192 gfp_t gfp_mask) 7193 { 7194 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) { 7195 struct page_counter *fail; 7196 7197 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) { 7198 memcg->tcpmem_pressure = 0; 7199 return true; 7200 } 7201 memcg->tcpmem_pressure = 1; 7202 if (gfp_mask & __GFP_NOFAIL) { 7203 page_counter_charge(&memcg->tcpmem, nr_pages); 7204 return true; 7205 } 7206 return false; 7207 } 7208 7209 if (try_charge(memcg, gfp_mask, nr_pages) == 0) { 7210 mod_memcg_state(memcg, MEMCG_SOCK, nr_pages); 7211 return true; 7212 } 7213 7214 return false; 7215 } 7216 7217 /** 7218 * mem_cgroup_uncharge_skmem - uncharge socket memory 7219 * @memcg: memcg to uncharge 7220 * @nr_pages: number of pages to uncharge 7221 */ 7222 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages) 7223 { 7224 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) { 7225 page_counter_uncharge(&memcg->tcpmem, nr_pages); 7226 return; 7227 } 7228 7229 mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages); 7230 7231 refill_stock(memcg, nr_pages); 7232 } 7233 7234 static int __init cgroup_memory(char *s) 7235 { 7236 char *token; 7237 7238 while ((token = strsep(&s, ",")) != NULL) { 7239 if (!*token) 7240 continue; 7241 if (!strcmp(token, "nosocket")) 7242 cgroup_memory_nosocket = true; 7243 if (!strcmp(token, "nokmem")) 7244 cgroup_memory_nokmem = true; 7245 } 7246 return 1; 7247 } 7248 __setup("cgroup.memory=", cgroup_memory); 7249 7250 /* 7251 * subsys_initcall() for memory controller. 7252 * 7253 * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this 7254 * context because of lock dependencies (cgroup_lock -> cpu hotplug) but 7255 * basically everything that doesn't depend on a specific mem_cgroup structure 7256 * should be initialized from here. 7257 */ 7258 static int __init mem_cgroup_init(void) 7259 { 7260 int cpu, node; 7261 7262 /* 7263 * Currently s32 type (can refer to struct batched_lruvec_stat) is 7264 * used for per-memcg-per-cpu caching of per-node statistics. In order 7265 * to work fine, we should make sure that the overfill threshold can't 7266 * exceed S32_MAX / PAGE_SIZE. 7267 */ 7268 BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S32_MAX / PAGE_SIZE); 7269 7270 cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL, 7271 memcg_hotplug_cpu_dead); 7272 7273 for_each_possible_cpu(cpu) 7274 INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work, 7275 drain_local_stock); 7276 7277 for_each_node(node) { 7278 struct mem_cgroup_tree_per_node *rtpn; 7279 7280 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, 7281 node_online(node) ? node : NUMA_NO_NODE); 7282 7283 rtpn->rb_root = RB_ROOT; 7284 rtpn->rb_rightmost = NULL; 7285 spin_lock_init(&rtpn->lock); 7286 soft_limit_tree.rb_tree_per_node[node] = rtpn; 7287 } 7288 7289 return 0; 7290 } 7291 subsys_initcall(mem_cgroup_init); 7292 7293 #ifdef CONFIG_SWAP 7294 static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg) 7295 { 7296 while (!refcount_inc_not_zero(&memcg->id.ref)) { 7297 /* 7298 * The root cgroup cannot be destroyed, so it's refcount must 7299 * always be >= 1. 7300 */ 7301 if (WARN_ON_ONCE(memcg == root_mem_cgroup)) { 7302 VM_BUG_ON(1); 7303 break; 7304 } 7305 memcg = parent_mem_cgroup(memcg); 7306 if (!memcg) 7307 memcg = root_mem_cgroup; 7308 } 7309 return memcg; 7310 } 7311 7312 /** 7313 * mem_cgroup_swapout - transfer a memsw charge to swap 7314 * @folio: folio whose memsw charge to transfer 7315 * @entry: swap entry to move the charge to 7316 * 7317 * Transfer the memsw charge of @folio to @entry. 7318 */ 7319 void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry) 7320 { 7321 struct mem_cgroup *memcg, *swap_memcg; 7322 unsigned int nr_entries; 7323 unsigned short oldid; 7324 7325 VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); 7326 VM_BUG_ON_FOLIO(folio_ref_count(folio), folio); 7327 7328 if (mem_cgroup_disabled()) 7329 return; 7330 7331 if (!do_memsw_account()) 7332 return; 7333 7334 memcg = folio_memcg(folio); 7335 7336 VM_WARN_ON_ONCE_FOLIO(!memcg, folio); 7337 if (!memcg) 7338 return; 7339 7340 /* 7341 * In case the memcg owning these pages has been offlined and doesn't 7342 * have an ID allocated to it anymore, charge the closest online 7343 * ancestor for the swap instead and transfer the memory+swap charge. 7344 */ 7345 swap_memcg = mem_cgroup_id_get_online(memcg); 7346 nr_entries = folio_nr_pages(folio); 7347 /* Get references for the tail pages, too */ 7348 if (nr_entries > 1) 7349 mem_cgroup_id_get_many(swap_memcg, nr_entries - 1); 7350 oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg), 7351 nr_entries); 7352 VM_BUG_ON_FOLIO(oldid, folio); 7353 mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries); 7354 7355 folio->memcg_data = 0; 7356 7357 if (!mem_cgroup_is_root(memcg)) 7358 page_counter_uncharge(&memcg->memory, nr_entries); 7359 7360 if (memcg != swap_memcg) { 7361 if (!mem_cgroup_is_root(swap_memcg)) 7362 page_counter_charge(&swap_memcg->memsw, nr_entries); 7363 page_counter_uncharge(&memcg->memsw, nr_entries); 7364 } 7365 7366 /* 7367 * Interrupts should be disabled here because the caller holds the 7368 * i_pages lock which is taken with interrupts-off. It is 7369 * important here to have the interrupts disabled because it is the 7370 * only synchronisation we have for updating the per-CPU variables. 7371 */ 7372 memcg_stats_lock(); 7373 mem_cgroup_charge_statistics(memcg, -nr_entries); 7374 memcg_stats_unlock(); 7375 memcg_check_events(memcg, folio_nid(folio)); 7376 7377 css_put(&memcg->css); 7378 } 7379 7380 /** 7381 * __mem_cgroup_try_charge_swap - try charging swap space for a folio 7382 * @folio: folio being added to swap 7383 * @entry: swap entry to charge 7384 * 7385 * Try to charge @folio's memcg for the swap space at @entry. 7386 * 7387 * Returns 0 on success, -ENOMEM on failure. 7388 */ 7389 int __mem_cgroup_try_charge_swap(struct folio *folio, swp_entry_t entry) 7390 { 7391 unsigned int nr_pages = folio_nr_pages(folio); 7392 struct page_counter *counter; 7393 struct mem_cgroup *memcg; 7394 unsigned short oldid; 7395 7396 if (do_memsw_account()) 7397 return 0; 7398 7399 memcg = folio_memcg(folio); 7400 7401 VM_WARN_ON_ONCE_FOLIO(!memcg, folio); 7402 if (!memcg) 7403 return 0; 7404 7405 if (!entry.val) { 7406 memcg_memory_event(memcg, MEMCG_SWAP_FAIL); 7407 return 0; 7408 } 7409 7410 memcg = mem_cgroup_id_get_online(memcg); 7411 7412 if (!mem_cgroup_is_root(memcg) && 7413 !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) { 7414 memcg_memory_event(memcg, MEMCG_SWAP_MAX); 7415 memcg_memory_event(memcg, MEMCG_SWAP_FAIL); 7416 mem_cgroup_id_put(memcg); 7417 return -ENOMEM; 7418 } 7419 7420 /* Get references for the tail pages, too */ 7421 if (nr_pages > 1) 7422 mem_cgroup_id_get_many(memcg, nr_pages - 1); 7423 oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages); 7424 VM_BUG_ON_FOLIO(oldid, folio); 7425 mod_memcg_state(memcg, MEMCG_SWAP, nr_pages); 7426 7427 return 0; 7428 } 7429 7430 /** 7431 * __mem_cgroup_uncharge_swap - uncharge swap space 7432 * @entry: swap entry to uncharge 7433 * @nr_pages: the amount of swap space to uncharge 7434 */ 7435 void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages) 7436 { 7437 struct mem_cgroup *memcg; 7438 unsigned short id; 7439 7440 if (mem_cgroup_disabled()) 7441 return; 7442 7443 id = swap_cgroup_record(entry, 0, nr_pages); 7444 rcu_read_lock(); 7445 memcg = mem_cgroup_from_id(id); 7446 if (memcg) { 7447 if (!mem_cgroup_is_root(memcg)) { 7448 if (do_memsw_account()) 7449 page_counter_uncharge(&memcg->memsw, nr_pages); 7450 else 7451 page_counter_uncharge(&memcg->swap, nr_pages); 7452 } 7453 mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages); 7454 mem_cgroup_id_put_many(memcg, nr_pages); 7455 } 7456 rcu_read_unlock(); 7457 } 7458 7459 long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg) 7460 { 7461 long nr_swap_pages = get_nr_swap_pages(); 7462 7463 if (mem_cgroup_disabled() || do_memsw_account()) 7464 return nr_swap_pages; 7465 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) 7466 nr_swap_pages = min_t(long, nr_swap_pages, 7467 READ_ONCE(memcg->swap.max) - 7468 page_counter_read(&memcg->swap)); 7469 return nr_swap_pages; 7470 } 7471 7472 bool mem_cgroup_swap_full(struct folio *folio) 7473 { 7474 struct mem_cgroup *memcg; 7475 7476 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 7477 7478 if (vm_swap_full()) 7479 return true; 7480 if (do_memsw_account()) 7481 return false; 7482 7483 memcg = folio_memcg(folio); 7484 if (!memcg) 7485 return false; 7486 7487 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) { 7488 unsigned long usage = page_counter_read(&memcg->swap); 7489 7490 if (usage * 2 >= READ_ONCE(memcg->swap.high) || 7491 usage * 2 >= READ_ONCE(memcg->swap.max)) 7492 return true; 7493 } 7494 7495 return false; 7496 } 7497 7498 static int __init setup_swap_account(char *s) 7499 { 7500 pr_warn_once("The swapaccount= commandline option is deprecated. " 7501 "Please report your usecase to linux-mm@kvack.org if you " 7502 "depend on this functionality.\n"); 7503 return 1; 7504 } 7505 __setup("swapaccount=", setup_swap_account); 7506 7507 static u64 swap_current_read(struct cgroup_subsys_state *css, 7508 struct cftype *cft) 7509 { 7510 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 7511 7512 return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE; 7513 } 7514 7515 static int swap_high_show(struct seq_file *m, void *v) 7516 { 7517 return seq_puts_memcg_tunable(m, 7518 READ_ONCE(mem_cgroup_from_seq(m)->swap.high)); 7519 } 7520 7521 static ssize_t swap_high_write(struct kernfs_open_file *of, 7522 char *buf, size_t nbytes, loff_t off) 7523 { 7524 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 7525 unsigned long high; 7526 int err; 7527 7528 buf = strstrip(buf); 7529 err = page_counter_memparse(buf, "max", &high); 7530 if (err) 7531 return err; 7532 7533 page_counter_set_high(&memcg->swap, high); 7534 7535 return nbytes; 7536 } 7537 7538 static int swap_max_show(struct seq_file *m, void *v) 7539 { 7540 return seq_puts_memcg_tunable(m, 7541 READ_ONCE(mem_cgroup_from_seq(m)->swap.max)); 7542 } 7543 7544 static ssize_t swap_max_write(struct kernfs_open_file *of, 7545 char *buf, size_t nbytes, loff_t off) 7546 { 7547 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 7548 unsigned long max; 7549 int err; 7550 7551 buf = strstrip(buf); 7552 err = page_counter_memparse(buf, "max", &max); 7553 if (err) 7554 return err; 7555 7556 xchg(&memcg->swap.max, max); 7557 7558 return nbytes; 7559 } 7560 7561 static int swap_events_show(struct seq_file *m, void *v) 7562 { 7563 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 7564 7565 seq_printf(m, "high %lu\n", 7566 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_HIGH])); 7567 seq_printf(m, "max %lu\n", 7568 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX])); 7569 seq_printf(m, "fail %lu\n", 7570 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL])); 7571 7572 return 0; 7573 } 7574 7575 static struct cftype swap_files[] = { 7576 { 7577 .name = "swap.current", 7578 .flags = CFTYPE_NOT_ON_ROOT, 7579 .read_u64 = swap_current_read, 7580 }, 7581 { 7582 .name = "swap.high", 7583 .flags = CFTYPE_NOT_ON_ROOT, 7584 .seq_show = swap_high_show, 7585 .write = swap_high_write, 7586 }, 7587 { 7588 .name = "swap.max", 7589 .flags = CFTYPE_NOT_ON_ROOT, 7590 .seq_show = swap_max_show, 7591 .write = swap_max_write, 7592 }, 7593 { 7594 .name = "swap.events", 7595 .flags = CFTYPE_NOT_ON_ROOT, 7596 .file_offset = offsetof(struct mem_cgroup, swap_events_file), 7597 .seq_show = swap_events_show, 7598 }, 7599 { } /* terminate */ 7600 }; 7601 7602 static struct cftype memsw_files[] = { 7603 { 7604 .name = "memsw.usage_in_bytes", 7605 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE), 7606 .read_u64 = mem_cgroup_read_u64, 7607 }, 7608 { 7609 .name = "memsw.max_usage_in_bytes", 7610 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE), 7611 .write = mem_cgroup_reset, 7612 .read_u64 = mem_cgroup_read_u64, 7613 }, 7614 { 7615 .name = "memsw.limit_in_bytes", 7616 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT), 7617 .write = mem_cgroup_write, 7618 .read_u64 = mem_cgroup_read_u64, 7619 }, 7620 { 7621 .name = "memsw.failcnt", 7622 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT), 7623 .write = mem_cgroup_reset, 7624 .read_u64 = mem_cgroup_read_u64, 7625 }, 7626 { }, /* terminate */ 7627 }; 7628 7629 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP) 7630 /** 7631 * obj_cgroup_may_zswap - check if this cgroup can zswap 7632 * @objcg: the object cgroup 7633 * 7634 * Check if the hierarchical zswap limit has been reached. 7635 * 7636 * This doesn't check for specific headroom, and it is not atomic 7637 * either. But with zswap, the size of the allocation is only known 7638 * once compression has occured, and this optimistic pre-check avoids 7639 * spending cycles on compression when there is already no room left 7640 * or zswap is disabled altogether somewhere in the hierarchy. 7641 */ 7642 bool obj_cgroup_may_zswap(struct obj_cgroup *objcg) 7643 { 7644 struct mem_cgroup *memcg, *original_memcg; 7645 bool ret = true; 7646 7647 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) 7648 return true; 7649 7650 original_memcg = get_mem_cgroup_from_objcg(objcg); 7651 for (memcg = original_memcg; memcg != root_mem_cgroup; 7652 memcg = parent_mem_cgroup(memcg)) { 7653 unsigned long max = READ_ONCE(memcg->zswap_max); 7654 unsigned long pages; 7655 7656 if (max == PAGE_COUNTER_MAX) 7657 continue; 7658 if (max == 0) { 7659 ret = false; 7660 break; 7661 } 7662 7663 cgroup_rstat_flush(memcg->css.cgroup); 7664 pages = memcg_page_state(memcg, MEMCG_ZSWAP_B) / PAGE_SIZE; 7665 if (pages < max) 7666 continue; 7667 ret = false; 7668 break; 7669 } 7670 mem_cgroup_put(original_memcg); 7671 return ret; 7672 } 7673 7674 /** 7675 * obj_cgroup_charge_zswap - charge compression backend memory 7676 * @objcg: the object cgroup 7677 * @size: size of compressed object 7678 * 7679 * This forces the charge after obj_cgroup_may_swap() allowed 7680 * compression and storage in zwap for this cgroup to go ahead. 7681 */ 7682 void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size) 7683 { 7684 struct mem_cgroup *memcg; 7685 7686 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) 7687 return; 7688 7689 VM_WARN_ON_ONCE(!(current->flags & PF_MEMALLOC)); 7690 7691 /* PF_MEMALLOC context, charging must succeed */ 7692 if (obj_cgroup_charge(objcg, GFP_KERNEL, size)) 7693 VM_WARN_ON_ONCE(1); 7694 7695 rcu_read_lock(); 7696 memcg = obj_cgroup_memcg(objcg); 7697 mod_memcg_state(memcg, MEMCG_ZSWAP_B, size); 7698 mod_memcg_state(memcg, MEMCG_ZSWAPPED, 1); 7699 rcu_read_unlock(); 7700 } 7701 7702 /** 7703 * obj_cgroup_uncharge_zswap - uncharge compression backend memory 7704 * @objcg: the object cgroup 7705 * @size: size of compressed object 7706 * 7707 * Uncharges zswap memory on page in. 7708 */ 7709 void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size) 7710 { 7711 struct mem_cgroup *memcg; 7712 7713 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) 7714 return; 7715 7716 obj_cgroup_uncharge(objcg, size); 7717 7718 rcu_read_lock(); 7719 memcg = obj_cgroup_memcg(objcg); 7720 mod_memcg_state(memcg, MEMCG_ZSWAP_B, -size); 7721 mod_memcg_state(memcg, MEMCG_ZSWAPPED, -1); 7722 rcu_read_unlock(); 7723 } 7724 7725 static u64 zswap_current_read(struct cgroup_subsys_state *css, 7726 struct cftype *cft) 7727 { 7728 cgroup_rstat_flush(css->cgroup); 7729 return memcg_page_state(mem_cgroup_from_css(css), MEMCG_ZSWAP_B); 7730 } 7731 7732 static int zswap_max_show(struct seq_file *m, void *v) 7733 { 7734 return seq_puts_memcg_tunable(m, 7735 READ_ONCE(mem_cgroup_from_seq(m)->zswap_max)); 7736 } 7737 7738 static ssize_t zswap_max_write(struct kernfs_open_file *of, 7739 char *buf, size_t nbytes, loff_t off) 7740 { 7741 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 7742 unsigned long max; 7743 int err; 7744 7745 buf = strstrip(buf); 7746 err = page_counter_memparse(buf, "max", &max); 7747 if (err) 7748 return err; 7749 7750 xchg(&memcg->zswap_max, max); 7751 7752 return nbytes; 7753 } 7754 7755 static struct cftype zswap_files[] = { 7756 { 7757 .name = "zswap.current", 7758 .flags = CFTYPE_NOT_ON_ROOT, 7759 .read_u64 = zswap_current_read, 7760 }, 7761 { 7762 .name = "zswap.max", 7763 .flags = CFTYPE_NOT_ON_ROOT, 7764 .seq_show = zswap_max_show, 7765 .write = zswap_max_write, 7766 }, 7767 { } /* terminate */ 7768 }; 7769 #endif /* CONFIG_MEMCG_KMEM && CONFIG_ZSWAP */ 7770 7771 static int __init mem_cgroup_swap_init(void) 7772 { 7773 if (mem_cgroup_disabled()) 7774 return 0; 7775 7776 WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, swap_files)); 7777 WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys, memsw_files)); 7778 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP) 7779 WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, zswap_files)); 7780 #endif 7781 return 0; 7782 } 7783 subsys_initcall(mem_cgroup_swap_init); 7784 7785 #endif /* CONFIG_SWAP */ 7786