1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* memcontrol.c - Memory Controller 3 * 4 * Copyright IBM Corporation, 2007 5 * Author Balbir Singh <balbir@linux.vnet.ibm.com> 6 * 7 * Copyright 2007 OpenVZ SWsoft Inc 8 * Author: Pavel Emelianov <xemul@openvz.org> 9 * 10 * Memory thresholds 11 * Copyright (C) 2009 Nokia Corporation 12 * Author: Kirill A. Shutemov 13 * 14 * Kernel Memory Controller 15 * Copyright (C) 2012 Parallels Inc. and Google Inc. 16 * Authors: Glauber Costa and Suleiman Souhlal 17 * 18 * Native page reclaim 19 * Charge lifetime sanitation 20 * Lockless page tracking & accounting 21 * Unified hierarchy configuration model 22 * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner 23 * 24 * Per memcg lru locking 25 * Copyright (C) 2020 Alibaba, Inc, Alex Shi 26 */ 27 28 #include <linux/page_counter.h> 29 #include <linux/memcontrol.h> 30 #include <linux/cgroup.h> 31 #include <linux/pagewalk.h> 32 #include <linux/sched/mm.h> 33 #include <linux/shmem_fs.h> 34 #include <linux/hugetlb.h> 35 #include <linux/pagemap.h> 36 #include <linux/vm_event_item.h> 37 #include <linux/smp.h> 38 #include <linux/page-flags.h> 39 #include <linux/backing-dev.h> 40 #include <linux/bit_spinlock.h> 41 #include <linux/rcupdate.h> 42 #include <linux/limits.h> 43 #include <linux/export.h> 44 #include <linux/mutex.h> 45 #include <linux/rbtree.h> 46 #include <linux/slab.h> 47 #include <linux/swap.h> 48 #include <linux/swapops.h> 49 #include <linux/spinlock.h> 50 #include <linux/eventfd.h> 51 #include <linux/poll.h> 52 #include <linux/sort.h> 53 #include <linux/fs.h> 54 #include <linux/seq_file.h> 55 #include <linux/vmpressure.h> 56 #include <linux/memremap.h> 57 #include <linux/mm_inline.h> 58 #include <linux/swap_cgroup.h> 59 #include <linux/cpu.h> 60 #include <linux/oom.h> 61 #include <linux/lockdep.h> 62 #include <linux/file.h> 63 #include <linux/resume_user_mode.h> 64 #include <linux/psi.h> 65 #include <linux/seq_buf.h> 66 #include <linux/sched/isolation.h> 67 #include "internal.h" 68 #include <net/sock.h> 69 #include <net/ip.h> 70 #include "slab.h" 71 #include "swap.h" 72 73 #include <linux/uaccess.h> 74 75 #include <trace/events/vmscan.h> 76 77 struct cgroup_subsys memory_cgrp_subsys __read_mostly; 78 EXPORT_SYMBOL(memory_cgrp_subsys); 79 80 struct mem_cgroup *root_mem_cgroup __read_mostly; 81 82 /* Active memory cgroup to use from an interrupt context */ 83 DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg); 84 EXPORT_PER_CPU_SYMBOL_GPL(int_active_memcg); 85 86 /* Socket memory accounting disabled? */ 87 static bool cgroup_memory_nosocket __ro_after_init; 88 89 /* Kernel memory accounting disabled? */ 90 static bool cgroup_memory_nokmem __ro_after_init; 91 92 /* BPF memory accounting disabled? */ 93 static bool cgroup_memory_nobpf __ro_after_init; 94 95 #ifdef CONFIG_CGROUP_WRITEBACK 96 static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq); 97 #endif 98 99 /* Whether legacy memory+swap accounting is active */ 100 static bool do_memsw_account(void) 101 { 102 return !cgroup_subsys_on_dfl(memory_cgrp_subsys); 103 } 104 105 #define THRESHOLDS_EVENTS_TARGET 128 106 #define SOFTLIMIT_EVENTS_TARGET 1024 107 108 /* 109 * Cgroups above their limits are maintained in a RB-Tree, independent of 110 * their hierarchy representation 111 */ 112 113 struct mem_cgroup_tree_per_node { 114 struct rb_root rb_root; 115 struct rb_node *rb_rightmost; 116 spinlock_t lock; 117 }; 118 119 struct mem_cgroup_tree { 120 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES]; 121 }; 122 123 static struct mem_cgroup_tree soft_limit_tree __read_mostly; 124 125 /* for OOM */ 126 struct mem_cgroup_eventfd_list { 127 struct list_head list; 128 struct eventfd_ctx *eventfd; 129 }; 130 131 /* 132 * cgroup_event represents events which userspace want to receive. 133 */ 134 struct mem_cgroup_event { 135 /* 136 * memcg which the event belongs to. 137 */ 138 struct mem_cgroup *memcg; 139 /* 140 * eventfd to signal userspace about the event. 141 */ 142 struct eventfd_ctx *eventfd; 143 /* 144 * Each of these stored in a list by the cgroup. 145 */ 146 struct list_head list; 147 /* 148 * register_event() callback will be used to add new userspace 149 * waiter for changes related to this event. Use eventfd_signal() 150 * on eventfd to send notification to userspace. 151 */ 152 int (*register_event)(struct mem_cgroup *memcg, 153 struct eventfd_ctx *eventfd, const char *args); 154 /* 155 * unregister_event() callback will be called when userspace closes 156 * the eventfd or on cgroup removing. This callback must be set, 157 * if you want provide notification functionality. 158 */ 159 void (*unregister_event)(struct mem_cgroup *memcg, 160 struct eventfd_ctx *eventfd); 161 /* 162 * All fields below needed to unregister event when 163 * userspace closes eventfd. 164 */ 165 poll_table pt; 166 wait_queue_head_t *wqh; 167 wait_queue_entry_t wait; 168 struct work_struct remove; 169 }; 170 171 static void mem_cgroup_threshold(struct mem_cgroup *memcg); 172 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg); 173 174 /* Stuffs for move charges at task migration. */ 175 /* 176 * Types of charges to be moved. 177 */ 178 #define MOVE_ANON 0x1U 179 #define MOVE_FILE 0x2U 180 #define MOVE_MASK (MOVE_ANON | MOVE_FILE) 181 182 /* "mc" and its members are protected by cgroup_mutex */ 183 static struct move_charge_struct { 184 spinlock_t lock; /* for from, to */ 185 struct mm_struct *mm; 186 struct mem_cgroup *from; 187 struct mem_cgroup *to; 188 unsigned long flags; 189 unsigned long precharge; 190 unsigned long moved_charge; 191 unsigned long moved_swap; 192 struct task_struct *moving_task; /* a task moving charges */ 193 wait_queue_head_t waitq; /* a waitq for other context */ 194 } mc = { 195 .lock = __SPIN_LOCK_UNLOCKED(mc.lock), 196 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq), 197 }; 198 199 /* 200 * Maximum loops in mem_cgroup_soft_reclaim(), used for soft 201 * limit reclaim to prevent infinite loops, if they ever occur. 202 */ 203 #define MEM_CGROUP_MAX_RECLAIM_LOOPS 100 204 #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2 205 206 /* for encoding cft->private value on file */ 207 enum res_type { 208 _MEM, 209 _MEMSWAP, 210 _KMEM, 211 _TCP, 212 }; 213 214 #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val)) 215 #define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff) 216 #define MEMFILE_ATTR(val) ((val) & 0xffff) 217 218 /* 219 * Iteration constructs for visiting all cgroups (under a tree). If 220 * loops are exited prematurely (break), mem_cgroup_iter_break() must 221 * be used for reference counting. 222 */ 223 #define for_each_mem_cgroup_tree(iter, root) \ 224 for (iter = mem_cgroup_iter(root, NULL, NULL); \ 225 iter != NULL; \ 226 iter = mem_cgroup_iter(root, iter, NULL)) 227 228 #define for_each_mem_cgroup(iter) \ 229 for (iter = mem_cgroup_iter(NULL, NULL, NULL); \ 230 iter != NULL; \ 231 iter = mem_cgroup_iter(NULL, iter, NULL)) 232 233 static inline bool task_is_dying(void) 234 { 235 return tsk_is_oom_victim(current) || fatal_signal_pending(current) || 236 (current->flags & PF_EXITING); 237 } 238 239 /* Some nice accessors for the vmpressure. */ 240 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg) 241 { 242 if (!memcg) 243 memcg = root_mem_cgroup; 244 return &memcg->vmpressure; 245 } 246 247 struct mem_cgroup *vmpressure_to_memcg(struct vmpressure *vmpr) 248 { 249 return container_of(vmpr, struct mem_cgroup, vmpressure); 250 } 251 252 #ifdef CONFIG_MEMCG_KMEM 253 static DEFINE_SPINLOCK(objcg_lock); 254 255 bool mem_cgroup_kmem_disabled(void) 256 { 257 return cgroup_memory_nokmem; 258 } 259 260 static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg, 261 unsigned int nr_pages); 262 263 static void obj_cgroup_release(struct percpu_ref *ref) 264 { 265 struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt); 266 unsigned int nr_bytes; 267 unsigned int nr_pages; 268 unsigned long flags; 269 270 /* 271 * At this point all allocated objects are freed, and 272 * objcg->nr_charged_bytes can't have an arbitrary byte value. 273 * However, it can be PAGE_SIZE or (x * PAGE_SIZE). 274 * 275 * The following sequence can lead to it: 276 * 1) CPU0: objcg == stock->cached_objcg 277 * 2) CPU1: we do a small allocation (e.g. 92 bytes), 278 * PAGE_SIZE bytes are charged 279 * 3) CPU1: a process from another memcg is allocating something, 280 * the stock if flushed, 281 * objcg->nr_charged_bytes = PAGE_SIZE - 92 282 * 5) CPU0: we do release this object, 283 * 92 bytes are added to stock->nr_bytes 284 * 6) CPU0: stock is flushed, 285 * 92 bytes are added to objcg->nr_charged_bytes 286 * 287 * In the result, nr_charged_bytes == PAGE_SIZE. 288 * This page will be uncharged in obj_cgroup_release(). 289 */ 290 nr_bytes = atomic_read(&objcg->nr_charged_bytes); 291 WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1)); 292 nr_pages = nr_bytes >> PAGE_SHIFT; 293 294 if (nr_pages) 295 obj_cgroup_uncharge_pages(objcg, nr_pages); 296 297 spin_lock_irqsave(&objcg_lock, flags); 298 list_del(&objcg->list); 299 spin_unlock_irqrestore(&objcg_lock, flags); 300 301 percpu_ref_exit(ref); 302 kfree_rcu(objcg, rcu); 303 } 304 305 static struct obj_cgroup *obj_cgroup_alloc(void) 306 { 307 struct obj_cgroup *objcg; 308 int ret; 309 310 objcg = kzalloc(sizeof(struct obj_cgroup), GFP_KERNEL); 311 if (!objcg) 312 return NULL; 313 314 ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0, 315 GFP_KERNEL); 316 if (ret) { 317 kfree(objcg); 318 return NULL; 319 } 320 INIT_LIST_HEAD(&objcg->list); 321 return objcg; 322 } 323 324 static void memcg_reparent_objcgs(struct mem_cgroup *memcg, 325 struct mem_cgroup *parent) 326 { 327 struct obj_cgroup *objcg, *iter; 328 329 objcg = rcu_replace_pointer(memcg->objcg, NULL, true); 330 331 spin_lock_irq(&objcg_lock); 332 333 /* 1) Ready to reparent active objcg. */ 334 list_add(&objcg->list, &memcg->objcg_list); 335 /* 2) Reparent active objcg and already reparented objcgs to parent. */ 336 list_for_each_entry(iter, &memcg->objcg_list, list) 337 WRITE_ONCE(iter->memcg, parent); 338 /* 3) Move already reparented objcgs to the parent's list */ 339 list_splice(&memcg->objcg_list, &parent->objcg_list); 340 341 spin_unlock_irq(&objcg_lock); 342 343 percpu_ref_kill(&objcg->refcnt); 344 } 345 346 /* 347 * A lot of the calls to the cache allocation functions are expected to be 348 * inlined by the compiler. Since the calls to memcg_slab_pre_alloc_hook() are 349 * conditional to this static branch, we'll have to allow modules that does 350 * kmem_cache_alloc and the such to see this symbol as well 351 */ 352 DEFINE_STATIC_KEY_FALSE(memcg_kmem_online_key); 353 EXPORT_SYMBOL(memcg_kmem_online_key); 354 355 DEFINE_STATIC_KEY_FALSE(memcg_bpf_enabled_key); 356 EXPORT_SYMBOL(memcg_bpf_enabled_key); 357 #endif 358 359 /** 360 * mem_cgroup_css_from_folio - css of the memcg associated with a folio 361 * @folio: folio of interest 362 * 363 * If memcg is bound to the default hierarchy, css of the memcg associated 364 * with @folio is returned. The returned css remains associated with @folio 365 * until it is released. 366 * 367 * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup 368 * is returned. 369 */ 370 struct cgroup_subsys_state *mem_cgroup_css_from_folio(struct folio *folio) 371 { 372 struct mem_cgroup *memcg = folio_memcg(folio); 373 374 if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys)) 375 memcg = root_mem_cgroup; 376 377 return &memcg->css; 378 } 379 380 /** 381 * page_cgroup_ino - return inode number of the memcg a page is charged to 382 * @page: the page 383 * 384 * Look up the closest online ancestor of the memory cgroup @page is charged to 385 * and return its inode number or 0 if @page is not charged to any cgroup. It 386 * is safe to call this function without holding a reference to @page. 387 * 388 * Note, this function is inherently racy, because there is nothing to prevent 389 * the cgroup inode from getting torn down and potentially reallocated a moment 390 * after page_cgroup_ino() returns, so it only should be used by callers that 391 * do not care (such as procfs interfaces). 392 */ 393 ino_t page_cgroup_ino(struct page *page) 394 { 395 struct mem_cgroup *memcg; 396 unsigned long ino = 0; 397 398 rcu_read_lock(); 399 /* page_folio() is racy here, but the entire function is racy anyway */ 400 memcg = folio_memcg_check(page_folio(page)); 401 402 while (memcg && !(memcg->css.flags & CSS_ONLINE)) 403 memcg = parent_mem_cgroup(memcg); 404 if (memcg) 405 ino = cgroup_ino(memcg->css.cgroup); 406 rcu_read_unlock(); 407 return ino; 408 } 409 410 static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz, 411 struct mem_cgroup_tree_per_node *mctz, 412 unsigned long new_usage_in_excess) 413 { 414 struct rb_node **p = &mctz->rb_root.rb_node; 415 struct rb_node *parent = NULL; 416 struct mem_cgroup_per_node *mz_node; 417 bool rightmost = true; 418 419 if (mz->on_tree) 420 return; 421 422 mz->usage_in_excess = new_usage_in_excess; 423 if (!mz->usage_in_excess) 424 return; 425 while (*p) { 426 parent = *p; 427 mz_node = rb_entry(parent, struct mem_cgroup_per_node, 428 tree_node); 429 if (mz->usage_in_excess < mz_node->usage_in_excess) { 430 p = &(*p)->rb_left; 431 rightmost = false; 432 } else { 433 p = &(*p)->rb_right; 434 } 435 } 436 437 if (rightmost) 438 mctz->rb_rightmost = &mz->tree_node; 439 440 rb_link_node(&mz->tree_node, parent, p); 441 rb_insert_color(&mz->tree_node, &mctz->rb_root); 442 mz->on_tree = true; 443 } 444 445 static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz, 446 struct mem_cgroup_tree_per_node *mctz) 447 { 448 if (!mz->on_tree) 449 return; 450 451 if (&mz->tree_node == mctz->rb_rightmost) 452 mctz->rb_rightmost = rb_prev(&mz->tree_node); 453 454 rb_erase(&mz->tree_node, &mctz->rb_root); 455 mz->on_tree = false; 456 } 457 458 static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz, 459 struct mem_cgroup_tree_per_node *mctz) 460 { 461 unsigned long flags; 462 463 spin_lock_irqsave(&mctz->lock, flags); 464 __mem_cgroup_remove_exceeded(mz, mctz); 465 spin_unlock_irqrestore(&mctz->lock, flags); 466 } 467 468 static unsigned long soft_limit_excess(struct mem_cgroup *memcg) 469 { 470 unsigned long nr_pages = page_counter_read(&memcg->memory); 471 unsigned long soft_limit = READ_ONCE(memcg->soft_limit); 472 unsigned long excess = 0; 473 474 if (nr_pages > soft_limit) 475 excess = nr_pages - soft_limit; 476 477 return excess; 478 } 479 480 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, int nid) 481 { 482 unsigned long excess; 483 struct mem_cgroup_per_node *mz; 484 struct mem_cgroup_tree_per_node *mctz; 485 486 if (lru_gen_enabled()) { 487 if (soft_limit_excess(memcg)) 488 lru_gen_soft_reclaim(memcg, nid); 489 return; 490 } 491 492 mctz = soft_limit_tree.rb_tree_per_node[nid]; 493 if (!mctz) 494 return; 495 /* 496 * Necessary to update all ancestors when hierarchy is used. 497 * because their event counter is not touched. 498 */ 499 for (; memcg; memcg = parent_mem_cgroup(memcg)) { 500 mz = memcg->nodeinfo[nid]; 501 excess = soft_limit_excess(memcg); 502 /* 503 * We have to update the tree if mz is on RB-tree or 504 * mem is over its softlimit. 505 */ 506 if (excess || mz->on_tree) { 507 unsigned long flags; 508 509 spin_lock_irqsave(&mctz->lock, flags); 510 /* if on-tree, remove it */ 511 if (mz->on_tree) 512 __mem_cgroup_remove_exceeded(mz, mctz); 513 /* 514 * Insert again. mz->usage_in_excess will be updated. 515 * If excess is 0, no tree ops. 516 */ 517 __mem_cgroup_insert_exceeded(mz, mctz, excess); 518 spin_unlock_irqrestore(&mctz->lock, flags); 519 } 520 } 521 } 522 523 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg) 524 { 525 struct mem_cgroup_tree_per_node *mctz; 526 struct mem_cgroup_per_node *mz; 527 int nid; 528 529 for_each_node(nid) { 530 mz = memcg->nodeinfo[nid]; 531 mctz = soft_limit_tree.rb_tree_per_node[nid]; 532 if (mctz) 533 mem_cgroup_remove_exceeded(mz, mctz); 534 } 535 } 536 537 static struct mem_cgroup_per_node * 538 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz) 539 { 540 struct mem_cgroup_per_node *mz; 541 542 retry: 543 mz = NULL; 544 if (!mctz->rb_rightmost) 545 goto done; /* Nothing to reclaim from */ 546 547 mz = rb_entry(mctz->rb_rightmost, 548 struct mem_cgroup_per_node, tree_node); 549 /* 550 * Remove the node now but someone else can add it back, 551 * we will to add it back at the end of reclaim to its correct 552 * position in the tree. 553 */ 554 __mem_cgroup_remove_exceeded(mz, mctz); 555 if (!soft_limit_excess(mz->memcg) || 556 !css_tryget(&mz->memcg->css)) 557 goto retry; 558 done: 559 return mz; 560 } 561 562 static struct mem_cgroup_per_node * 563 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz) 564 { 565 struct mem_cgroup_per_node *mz; 566 567 spin_lock_irq(&mctz->lock); 568 mz = __mem_cgroup_largest_soft_limit_node(mctz); 569 spin_unlock_irq(&mctz->lock); 570 return mz; 571 } 572 573 /* 574 * memcg and lruvec stats flushing 575 * 576 * Many codepaths leading to stats update or read are performance sensitive and 577 * adding stats flushing in such codepaths is not desirable. So, to optimize the 578 * flushing the kernel does: 579 * 580 * 1) Periodically and asynchronously flush the stats every 2 seconds to not let 581 * rstat update tree grow unbounded. 582 * 583 * 2) Flush the stats synchronously on reader side only when there are more than 584 * (MEMCG_CHARGE_BATCH * nr_cpus) update events. Though this optimization 585 * will let stats be out of sync by atmost (MEMCG_CHARGE_BATCH * nr_cpus) but 586 * only for 2 seconds due to (1). 587 */ 588 static void flush_memcg_stats_dwork(struct work_struct *w); 589 static DECLARE_DEFERRABLE_WORK(stats_flush_dwork, flush_memcg_stats_dwork); 590 static DEFINE_PER_CPU(unsigned int, stats_updates); 591 static atomic_t stats_flush_ongoing = ATOMIC_INIT(0); 592 static atomic_t stats_flush_threshold = ATOMIC_INIT(0); 593 static u64 flush_next_time; 594 595 #define FLUSH_TIME (2UL*HZ) 596 597 /* 598 * Accessors to ensure that preemption is disabled on PREEMPT_RT because it can 599 * not rely on this as part of an acquired spinlock_t lock. These functions are 600 * never used in hardirq context on PREEMPT_RT and therefore disabling preemtion 601 * is sufficient. 602 */ 603 static void memcg_stats_lock(void) 604 { 605 preempt_disable_nested(); 606 VM_WARN_ON_IRQS_ENABLED(); 607 } 608 609 static void __memcg_stats_lock(void) 610 { 611 preempt_disable_nested(); 612 } 613 614 static void memcg_stats_unlock(void) 615 { 616 preempt_enable_nested(); 617 } 618 619 static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val) 620 { 621 unsigned int x; 622 623 if (!val) 624 return; 625 626 cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id()); 627 628 x = __this_cpu_add_return(stats_updates, abs(val)); 629 if (x > MEMCG_CHARGE_BATCH) { 630 /* 631 * If stats_flush_threshold exceeds the threshold 632 * (>num_online_cpus()), cgroup stats update will be triggered 633 * in __mem_cgroup_flush_stats(). Increasing this var further 634 * is redundant and simply adds overhead in atomic update. 635 */ 636 if (atomic_read(&stats_flush_threshold) <= num_online_cpus()) 637 atomic_add(x / MEMCG_CHARGE_BATCH, &stats_flush_threshold); 638 __this_cpu_write(stats_updates, 0); 639 } 640 } 641 642 static void do_flush_stats(void) 643 { 644 /* 645 * We always flush the entire tree, so concurrent flushers can just 646 * skip. This avoids a thundering herd problem on the rstat global lock 647 * from memcg flushers (e.g. reclaim, refault, etc). 648 */ 649 if (atomic_read(&stats_flush_ongoing) || 650 atomic_xchg(&stats_flush_ongoing, 1)) 651 return; 652 653 WRITE_ONCE(flush_next_time, jiffies_64 + 2*FLUSH_TIME); 654 655 cgroup_rstat_flush(root_mem_cgroup->css.cgroup); 656 657 atomic_set(&stats_flush_threshold, 0); 658 atomic_set(&stats_flush_ongoing, 0); 659 } 660 661 void mem_cgroup_flush_stats(void) 662 { 663 if (atomic_read(&stats_flush_threshold) > num_online_cpus()) 664 do_flush_stats(); 665 } 666 667 void mem_cgroup_flush_stats_ratelimited(void) 668 { 669 if (time_after64(jiffies_64, READ_ONCE(flush_next_time))) 670 mem_cgroup_flush_stats(); 671 } 672 673 static void flush_memcg_stats_dwork(struct work_struct *w) 674 { 675 /* 676 * Always flush here so that flushing in latency-sensitive paths is 677 * as cheap as possible. 678 */ 679 do_flush_stats(); 680 queue_delayed_work(system_unbound_wq, &stats_flush_dwork, FLUSH_TIME); 681 } 682 683 /* Subset of vm_event_item to report for memcg event stats */ 684 static const unsigned int memcg_vm_event_stat[] = { 685 PGPGIN, 686 PGPGOUT, 687 PGSCAN_KSWAPD, 688 PGSCAN_DIRECT, 689 PGSCAN_KHUGEPAGED, 690 PGSTEAL_KSWAPD, 691 PGSTEAL_DIRECT, 692 PGSTEAL_KHUGEPAGED, 693 PGFAULT, 694 PGMAJFAULT, 695 PGREFILL, 696 PGACTIVATE, 697 PGDEACTIVATE, 698 PGLAZYFREE, 699 PGLAZYFREED, 700 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP) 701 ZSWPIN, 702 ZSWPOUT, 703 #endif 704 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 705 THP_FAULT_ALLOC, 706 THP_COLLAPSE_ALLOC, 707 #endif 708 }; 709 710 #define NR_MEMCG_EVENTS ARRAY_SIZE(memcg_vm_event_stat) 711 static int mem_cgroup_events_index[NR_VM_EVENT_ITEMS] __read_mostly; 712 713 static void init_memcg_events(void) 714 { 715 int i; 716 717 for (i = 0; i < NR_MEMCG_EVENTS; ++i) 718 mem_cgroup_events_index[memcg_vm_event_stat[i]] = i + 1; 719 } 720 721 static inline int memcg_events_index(enum vm_event_item idx) 722 { 723 return mem_cgroup_events_index[idx] - 1; 724 } 725 726 struct memcg_vmstats_percpu { 727 /* Local (CPU and cgroup) page state & events */ 728 long state[MEMCG_NR_STAT]; 729 unsigned long events[NR_MEMCG_EVENTS]; 730 731 /* Delta calculation for lockless upward propagation */ 732 long state_prev[MEMCG_NR_STAT]; 733 unsigned long events_prev[NR_MEMCG_EVENTS]; 734 735 /* Cgroup1: threshold notifications & softlimit tree updates */ 736 unsigned long nr_page_events; 737 unsigned long targets[MEM_CGROUP_NTARGETS]; 738 }; 739 740 struct memcg_vmstats { 741 /* Aggregated (CPU and subtree) page state & events */ 742 long state[MEMCG_NR_STAT]; 743 unsigned long events[NR_MEMCG_EVENTS]; 744 745 /* Non-hierarchical (CPU aggregated) page state & events */ 746 long state_local[MEMCG_NR_STAT]; 747 unsigned long events_local[NR_MEMCG_EVENTS]; 748 749 /* Pending child counts during tree propagation */ 750 long state_pending[MEMCG_NR_STAT]; 751 unsigned long events_pending[NR_MEMCG_EVENTS]; 752 }; 753 754 unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx) 755 { 756 long x = READ_ONCE(memcg->vmstats->state[idx]); 757 #ifdef CONFIG_SMP 758 if (x < 0) 759 x = 0; 760 #endif 761 return x; 762 } 763 764 /** 765 * __mod_memcg_state - update cgroup memory statistics 766 * @memcg: the memory cgroup 767 * @idx: the stat item - can be enum memcg_stat_item or enum node_stat_item 768 * @val: delta to add to the counter, can be negative 769 */ 770 void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val) 771 { 772 if (mem_cgroup_disabled()) 773 return; 774 775 __this_cpu_add(memcg->vmstats_percpu->state[idx], val); 776 memcg_rstat_updated(memcg, val); 777 } 778 779 /* idx can be of type enum memcg_stat_item or node_stat_item. */ 780 static unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx) 781 { 782 long x = READ_ONCE(memcg->vmstats->state_local[idx]); 783 784 #ifdef CONFIG_SMP 785 if (x < 0) 786 x = 0; 787 #endif 788 return x; 789 } 790 791 void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, 792 int val) 793 { 794 struct mem_cgroup_per_node *pn; 795 struct mem_cgroup *memcg; 796 797 pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); 798 memcg = pn->memcg; 799 800 /* 801 * The caller from rmap relay on disabled preemption becase they never 802 * update their counter from in-interrupt context. For these two 803 * counters we check that the update is never performed from an 804 * interrupt context while other caller need to have disabled interrupt. 805 */ 806 __memcg_stats_lock(); 807 if (IS_ENABLED(CONFIG_DEBUG_VM)) { 808 switch (idx) { 809 case NR_ANON_MAPPED: 810 case NR_FILE_MAPPED: 811 case NR_ANON_THPS: 812 case NR_SHMEM_PMDMAPPED: 813 case NR_FILE_PMDMAPPED: 814 WARN_ON_ONCE(!in_task()); 815 break; 816 default: 817 VM_WARN_ON_IRQS_ENABLED(); 818 } 819 } 820 821 /* Update memcg */ 822 __this_cpu_add(memcg->vmstats_percpu->state[idx], val); 823 824 /* Update lruvec */ 825 __this_cpu_add(pn->lruvec_stats_percpu->state[idx], val); 826 827 memcg_rstat_updated(memcg, val); 828 memcg_stats_unlock(); 829 } 830 831 /** 832 * __mod_lruvec_state - update lruvec memory statistics 833 * @lruvec: the lruvec 834 * @idx: the stat item 835 * @val: delta to add to the counter, can be negative 836 * 837 * The lruvec is the intersection of the NUMA node and a cgroup. This 838 * function updates the all three counters that are affected by a 839 * change of state at this level: per-node, per-cgroup, per-lruvec. 840 */ 841 void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, 842 int val) 843 { 844 /* Update node */ 845 __mod_node_page_state(lruvec_pgdat(lruvec), idx, val); 846 847 /* Update memcg and lruvec */ 848 if (!mem_cgroup_disabled()) 849 __mod_memcg_lruvec_state(lruvec, idx, val); 850 } 851 852 void __mod_lruvec_page_state(struct page *page, enum node_stat_item idx, 853 int val) 854 { 855 struct page *head = compound_head(page); /* rmap on tail pages */ 856 struct mem_cgroup *memcg; 857 pg_data_t *pgdat = page_pgdat(page); 858 struct lruvec *lruvec; 859 860 rcu_read_lock(); 861 memcg = page_memcg(head); 862 /* Untracked pages have no memcg, no lruvec. Update only the node */ 863 if (!memcg) { 864 rcu_read_unlock(); 865 __mod_node_page_state(pgdat, idx, val); 866 return; 867 } 868 869 lruvec = mem_cgroup_lruvec(memcg, pgdat); 870 __mod_lruvec_state(lruvec, idx, val); 871 rcu_read_unlock(); 872 } 873 EXPORT_SYMBOL(__mod_lruvec_page_state); 874 875 void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val) 876 { 877 pg_data_t *pgdat = page_pgdat(virt_to_page(p)); 878 struct mem_cgroup *memcg; 879 struct lruvec *lruvec; 880 881 rcu_read_lock(); 882 memcg = mem_cgroup_from_slab_obj(p); 883 884 /* 885 * Untracked pages have no memcg, no lruvec. Update only the 886 * node. If we reparent the slab objects to the root memcg, 887 * when we free the slab object, we need to update the per-memcg 888 * vmstats to keep it correct for the root memcg. 889 */ 890 if (!memcg) { 891 __mod_node_page_state(pgdat, idx, val); 892 } else { 893 lruvec = mem_cgroup_lruvec(memcg, pgdat); 894 __mod_lruvec_state(lruvec, idx, val); 895 } 896 rcu_read_unlock(); 897 } 898 899 /** 900 * __count_memcg_events - account VM events in a cgroup 901 * @memcg: the memory cgroup 902 * @idx: the event item 903 * @count: the number of events that occurred 904 */ 905 void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, 906 unsigned long count) 907 { 908 int index = memcg_events_index(idx); 909 910 if (mem_cgroup_disabled() || index < 0) 911 return; 912 913 memcg_stats_lock(); 914 __this_cpu_add(memcg->vmstats_percpu->events[index], count); 915 memcg_rstat_updated(memcg, count); 916 memcg_stats_unlock(); 917 } 918 919 static unsigned long memcg_events(struct mem_cgroup *memcg, int event) 920 { 921 int index = memcg_events_index(event); 922 923 if (index < 0) 924 return 0; 925 return READ_ONCE(memcg->vmstats->events[index]); 926 } 927 928 static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event) 929 { 930 int index = memcg_events_index(event); 931 932 if (index < 0) 933 return 0; 934 935 return READ_ONCE(memcg->vmstats->events_local[index]); 936 } 937 938 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg, 939 int nr_pages) 940 { 941 /* pagein of a big page is an event. So, ignore page size */ 942 if (nr_pages > 0) 943 __count_memcg_events(memcg, PGPGIN, 1); 944 else { 945 __count_memcg_events(memcg, PGPGOUT, 1); 946 nr_pages = -nr_pages; /* for event */ 947 } 948 949 __this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages); 950 } 951 952 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg, 953 enum mem_cgroup_events_target target) 954 { 955 unsigned long val, next; 956 957 val = __this_cpu_read(memcg->vmstats_percpu->nr_page_events); 958 next = __this_cpu_read(memcg->vmstats_percpu->targets[target]); 959 /* from time_after() in jiffies.h */ 960 if ((long)(next - val) < 0) { 961 switch (target) { 962 case MEM_CGROUP_TARGET_THRESH: 963 next = val + THRESHOLDS_EVENTS_TARGET; 964 break; 965 case MEM_CGROUP_TARGET_SOFTLIMIT: 966 next = val + SOFTLIMIT_EVENTS_TARGET; 967 break; 968 default: 969 break; 970 } 971 __this_cpu_write(memcg->vmstats_percpu->targets[target], next); 972 return true; 973 } 974 return false; 975 } 976 977 /* 978 * Check events in order. 979 * 980 */ 981 static void memcg_check_events(struct mem_cgroup *memcg, int nid) 982 { 983 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 984 return; 985 986 /* threshold event is triggered in finer grain than soft limit */ 987 if (unlikely(mem_cgroup_event_ratelimit(memcg, 988 MEM_CGROUP_TARGET_THRESH))) { 989 bool do_softlimit; 990 991 do_softlimit = mem_cgroup_event_ratelimit(memcg, 992 MEM_CGROUP_TARGET_SOFTLIMIT); 993 mem_cgroup_threshold(memcg); 994 if (unlikely(do_softlimit)) 995 mem_cgroup_update_tree(memcg, nid); 996 } 997 } 998 999 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) 1000 { 1001 /* 1002 * mm_update_next_owner() may clear mm->owner to NULL 1003 * if it races with swapoff, page migration, etc. 1004 * So this can be called with p == NULL. 1005 */ 1006 if (unlikely(!p)) 1007 return NULL; 1008 1009 return mem_cgroup_from_css(task_css(p, memory_cgrp_id)); 1010 } 1011 EXPORT_SYMBOL(mem_cgroup_from_task); 1012 1013 static __always_inline struct mem_cgroup *active_memcg(void) 1014 { 1015 if (!in_task()) 1016 return this_cpu_read(int_active_memcg); 1017 else 1018 return current->active_memcg; 1019 } 1020 1021 /** 1022 * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg. 1023 * @mm: mm from which memcg should be extracted. It can be NULL. 1024 * 1025 * Obtain a reference on mm->memcg and returns it if successful. If mm 1026 * is NULL, then the memcg is chosen as follows: 1027 * 1) The active memcg, if set. 1028 * 2) current->mm->memcg, if available 1029 * 3) root memcg 1030 * If mem_cgroup is disabled, NULL is returned. 1031 */ 1032 struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm) 1033 { 1034 struct mem_cgroup *memcg; 1035 1036 if (mem_cgroup_disabled()) 1037 return NULL; 1038 1039 /* 1040 * Page cache insertions can happen without an 1041 * actual mm context, e.g. during disk probing 1042 * on boot, loopback IO, acct() writes etc. 1043 * 1044 * No need to css_get on root memcg as the reference 1045 * counting is disabled on the root level in the 1046 * cgroup core. See CSS_NO_REF. 1047 */ 1048 if (unlikely(!mm)) { 1049 memcg = active_memcg(); 1050 if (unlikely(memcg)) { 1051 /* remote memcg must hold a ref */ 1052 css_get(&memcg->css); 1053 return memcg; 1054 } 1055 mm = current->mm; 1056 if (unlikely(!mm)) 1057 return root_mem_cgroup; 1058 } 1059 1060 rcu_read_lock(); 1061 do { 1062 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); 1063 if (unlikely(!memcg)) 1064 memcg = root_mem_cgroup; 1065 } while (!css_tryget(&memcg->css)); 1066 rcu_read_unlock(); 1067 return memcg; 1068 } 1069 EXPORT_SYMBOL(get_mem_cgroup_from_mm); 1070 1071 static __always_inline bool memcg_kmem_bypass(void) 1072 { 1073 /* Allow remote memcg charging from any context. */ 1074 if (unlikely(active_memcg())) 1075 return false; 1076 1077 /* Memcg to charge can't be determined. */ 1078 if (!in_task() || !current->mm || (current->flags & PF_KTHREAD)) 1079 return true; 1080 1081 return false; 1082 } 1083 1084 /** 1085 * mem_cgroup_iter - iterate over memory cgroup hierarchy 1086 * @root: hierarchy root 1087 * @prev: previously returned memcg, NULL on first invocation 1088 * @reclaim: cookie for shared reclaim walks, NULL for full walks 1089 * 1090 * Returns references to children of the hierarchy below @root, or 1091 * @root itself, or %NULL after a full round-trip. 1092 * 1093 * Caller must pass the return value in @prev on subsequent 1094 * invocations for reference counting, or use mem_cgroup_iter_break() 1095 * to cancel a hierarchy walk before the round-trip is complete. 1096 * 1097 * Reclaimers can specify a node in @reclaim to divide up the memcgs 1098 * in the hierarchy among all concurrent reclaimers operating on the 1099 * same node. 1100 */ 1101 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root, 1102 struct mem_cgroup *prev, 1103 struct mem_cgroup_reclaim_cookie *reclaim) 1104 { 1105 struct mem_cgroup_reclaim_iter *iter; 1106 struct cgroup_subsys_state *css = NULL; 1107 struct mem_cgroup *memcg = NULL; 1108 struct mem_cgroup *pos = NULL; 1109 1110 if (mem_cgroup_disabled()) 1111 return NULL; 1112 1113 if (!root) 1114 root = root_mem_cgroup; 1115 1116 rcu_read_lock(); 1117 1118 if (reclaim) { 1119 struct mem_cgroup_per_node *mz; 1120 1121 mz = root->nodeinfo[reclaim->pgdat->node_id]; 1122 iter = &mz->iter; 1123 1124 /* 1125 * On start, join the current reclaim iteration cycle. 1126 * Exit when a concurrent walker completes it. 1127 */ 1128 if (!prev) 1129 reclaim->generation = iter->generation; 1130 else if (reclaim->generation != iter->generation) 1131 goto out_unlock; 1132 1133 while (1) { 1134 pos = READ_ONCE(iter->position); 1135 if (!pos || css_tryget(&pos->css)) 1136 break; 1137 /* 1138 * css reference reached zero, so iter->position will 1139 * be cleared by ->css_released. However, we should not 1140 * rely on this happening soon, because ->css_released 1141 * is called from a work queue, and by busy-waiting we 1142 * might block it. So we clear iter->position right 1143 * away. 1144 */ 1145 (void)cmpxchg(&iter->position, pos, NULL); 1146 } 1147 } else if (prev) { 1148 pos = prev; 1149 } 1150 1151 if (pos) 1152 css = &pos->css; 1153 1154 for (;;) { 1155 css = css_next_descendant_pre(css, &root->css); 1156 if (!css) { 1157 /* 1158 * Reclaimers share the hierarchy walk, and a 1159 * new one might jump in right at the end of 1160 * the hierarchy - make sure they see at least 1161 * one group and restart from the beginning. 1162 */ 1163 if (!prev) 1164 continue; 1165 break; 1166 } 1167 1168 /* 1169 * Verify the css and acquire a reference. The root 1170 * is provided by the caller, so we know it's alive 1171 * and kicking, and don't take an extra reference. 1172 */ 1173 if (css == &root->css || css_tryget(css)) { 1174 memcg = mem_cgroup_from_css(css); 1175 break; 1176 } 1177 } 1178 1179 if (reclaim) { 1180 /* 1181 * The position could have already been updated by a competing 1182 * thread, so check that the value hasn't changed since we read 1183 * it to avoid reclaiming from the same cgroup twice. 1184 */ 1185 (void)cmpxchg(&iter->position, pos, memcg); 1186 1187 if (pos) 1188 css_put(&pos->css); 1189 1190 if (!memcg) 1191 iter->generation++; 1192 } 1193 1194 out_unlock: 1195 rcu_read_unlock(); 1196 if (prev && prev != root) 1197 css_put(&prev->css); 1198 1199 return memcg; 1200 } 1201 1202 /** 1203 * mem_cgroup_iter_break - abort a hierarchy walk prematurely 1204 * @root: hierarchy root 1205 * @prev: last visited hierarchy member as returned by mem_cgroup_iter() 1206 */ 1207 void mem_cgroup_iter_break(struct mem_cgroup *root, 1208 struct mem_cgroup *prev) 1209 { 1210 if (!root) 1211 root = root_mem_cgroup; 1212 if (prev && prev != root) 1213 css_put(&prev->css); 1214 } 1215 1216 static void __invalidate_reclaim_iterators(struct mem_cgroup *from, 1217 struct mem_cgroup *dead_memcg) 1218 { 1219 struct mem_cgroup_reclaim_iter *iter; 1220 struct mem_cgroup_per_node *mz; 1221 int nid; 1222 1223 for_each_node(nid) { 1224 mz = from->nodeinfo[nid]; 1225 iter = &mz->iter; 1226 cmpxchg(&iter->position, dead_memcg, NULL); 1227 } 1228 } 1229 1230 static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg) 1231 { 1232 struct mem_cgroup *memcg = dead_memcg; 1233 struct mem_cgroup *last; 1234 1235 do { 1236 __invalidate_reclaim_iterators(memcg, dead_memcg); 1237 last = memcg; 1238 } while ((memcg = parent_mem_cgroup(memcg))); 1239 1240 /* 1241 * When cgroup1 non-hierarchy mode is used, 1242 * parent_mem_cgroup() does not walk all the way up to the 1243 * cgroup root (root_mem_cgroup). So we have to handle 1244 * dead_memcg from cgroup root separately. 1245 */ 1246 if (!mem_cgroup_is_root(last)) 1247 __invalidate_reclaim_iterators(root_mem_cgroup, 1248 dead_memcg); 1249 } 1250 1251 /** 1252 * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy 1253 * @memcg: hierarchy root 1254 * @fn: function to call for each task 1255 * @arg: argument passed to @fn 1256 * 1257 * This function iterates over tasks attached to @memcg or to any of its 1258 * descendants and calls @fn for each task. If @fn returns a non-zero 1259 * value, the function breaks the iteration loop. Otherwise, it will iterate 1260 * over all tasks and return 0. 1261 * 1262 * This function must not be called for the root memory cgroup. 1263 */ 1264 void mem_cgroup_scan_tasks(struct mem_cgroup *memcg, 1265 int (*fn)(struct task_struct *, void *), void *arg) 1266 { 1267 struct mem_cgroup *iter; 1268 int ret = 0; 1269 1270 BUG_ON(mem_cgroup_is_root(memcg)); 1271 1272 for_each_mem_cgroup_tree(iter, memcg) { 1273 struct css_task_iter it; 1274 struct task_struct *task; 1275 1276 css_task_iter_start(&iter->css, CSS_TASK_ITER_PROCS, &it); 1277 while (!ret && (task = css_task_iter_next(&it))) 1278 ret = fn(task, arg); 1279 css_task_iter_end(&it); 1280 if (ret) { 1281 mem_cgroup_iter_break(memcg, iter); 1282 break; 1283 } 1284 } 1285 } 1286 1287 #ifdef CONFIG_DEBUG_VM 1288 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio) 1289 { 1290 struct mem_cgroup *memcg; 1291 1292 if (mem_cgroup_disabled()) 1293 return; 1294 1295 memcg = folio_memcg(folio); 1296 1297 if (!memcg) 1298 VM_BUG_ON_FOLIO(!mem_cgroup_is_root(lruvec_memcg(lruvec)), folio); 1299 else 1300 VM_BUG_ON_FOLIO(lruvec_memcg(lruvec) != memcg, folio); 1301 } 1302 #endif 1303 1304 /** 1305 * folio_lruvec_lock - Lock the lruvec for a folio. 1306 * @folio: Pointer to the folio. 1307 * 1308 * These functions are safe to use under any of the following conditions: 1309 * - folio locked 1310 * - folio_test_lru false 1311 * - folio_memcg_lock() 1312 * - folio frozen (refcount of 0) 1313 * 1314 * Return: The lruvec this folio is on with its lock held. 1315 */ 1316 struct lruvec *folio_lruvec_lock(struct folio *folio) 1317 { 1318 struct lruvec *lruvec = folio_lruvec(folio); 1319 1320 spin_lock(&lruvec->lru_lock); 1321 lruvec_memcg_debug(lruvec, folio); 1322 1323 return lruvec; 1324 } 1325 1326 /** 1327 * folio_lruvec_lock_irq - Lock the lruvec for a folio. 1328 * @folio: Pointer to the folio. 1329 * 1330 * These functions are safe to use under any of the following conditions: 1331 * - folio locked 1332 * - folio_test_lru false 1333 * - folio_memcg_lock() 1334 * - folio frozen (refcount of 0) 1335 * 1336 * Return: The lruvec this folio is on with its lock held and interrupts 1337 * disabled. 1338 */ 1339 struct lruvec *folio_lruvec_lock_irq(struct folio *folio) 1340 { 1341 struct lruvec *lruvec = folio_lruvec(folio); 1342 1343 spin_lock_irq(&lruvec->lru_lock); 1344 lruvec_memcg_debug(lruvec, folio); 1345 1346 return lruvec; 1347 } 1348 1349 /** 1350 * folio_lruvec_lock_irqsave - Lock the lruvec for a folio. 1351 * @folio: Pointer to the folio. 1352 * @flags: Pointer to irqsave flags. 1353 * 1354 * These functions are safe to use under any of the following conditions: 1355 * - folio locked 1356 * - folio_test_lru false 1357 * - folio_memcg_lock() 1358 * - folio frozen (refcount of 0) 1359 * 1360 * Return: The lruvec this folio is on with its lock held and interrupts 1361 * disabled. 1362 */ 1363 struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio, 1364 unsigned long *flags) 1365 { 1366 struct lruvec *lruvec = folio_lruvec(folio); 1367 1368 spin_lock_irqsave(&lruvec->lru_lock, *flags); 1369 lruvec_memcg_debug(lruvec, folio); 1370 1371 return lruvec; 1372 } 1373 1374 /** 1375 * mem_cgroup_update_lru_size - account for adding or removing an lru page 1376 * @lruvec: mem_cgroup per zone lru vector 1377 * @lru: index of lru list the page is sitting on 1378 * @zid: zone id of the accounted pages 1379 * @nr_pages: positive when adding or negative when removing 1380 * 1381 * This function must be called under lru_lock, just before a page is added 1382 * to or just after a page is removed from an lru list. 1383 */ 1384 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, 1385 int zid, int nr_pages) 1386 { 1387 struct mem_cgroup_per_node *mz; 1388 unsigned long *lru_size; 1389 long size; 1390 1391 if (mem_cgroup_disabled()) 1392 return; 1393 1394 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); 1395 lru_size = &mz->lru_zone_size[zid][lru]; 1396 1397 if (nr_pages < 0) 1398 *lru_size += nr_pages; 1399 1400 size = *lru_size; 1401 if (WARN_ONCE(size < 0, 1402 "%s(%p, %d, %d): lru_size %ld\n", 1403 __func__, lruvec, lru, nr_pages, size)) { 1404 VM_BUG_ON(1); 1405 *lru_size = 0; 1406 } 1407 1408 if (nr_pages > 0) 1409 *lru_size += nr_pages; 1410 } 1411 1412 /** 1413 * mem_cgroup_margin - calculate chargeable space of a memory cgroup 1414 * @memcg: the memory cgroup 1415 * 1416 * Returns the maximum amount of memory @mem can be charged with, in 1417 * pages. 1418 */ 1419 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg) 1420 { 1421 unsigned long margin = 0; 1422 unsigned long count; 1423 unsigned long limit; 1424 1425 count = page_counter_read(&memcg->memory); 1426 limit = READ_ONCE(memcg->memory.max); 1427 if (count < limit) 1428 margin = limit - count; 1429 1430 if (do_memsw_account()) { 1431 count = page_counter_read(&memcg->memsw); 1432 limit = READ_ONCE(memcg->memsw.max); 1433 if (count < limit) 1434 margin = min(margin, limit - count); 1435 else 1436 margin = 0; 1437 } 1438 1439 return margin; 1440 } 1441 1442 /* 1443 * A routine for checking "mem" is under move_account() or not. 1444 * 1445 * Checking a cgroup is mc.from or mc.to or under hierarchy of 1446 * moving cgroups. This is for waiting at high-memory pressure 1447 * caused by "move". 1448 */ 1449 static bool mem_cgroup_under_move(struct mem_cgroup *memcg) 1450 { 1451 struct mem_cgroup *from; 1452 struct mem_cgroup *to; 1453 bool ret = false; 1454 /* 1455 * Unlike task_move routines, we access mc.to, mc.from not under 1456 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead. 1457 */ 1458 spin_lock(&mc.lock); 1459 from = mc.from; 1460 to = mc.to; 1461 if (!from) 1462 goto unlock; 1463 1464 ret = mem_cgroup_is_descendant(from, memcg) || 1465 mem_cgroup_is_descendant(to, memcg); 1466 unlock: 1467 spin_unlock(&mc.lock); 1468 return ret; 1469 } 1470 1471 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg) 1472 { 1473 if (mc.moving_task && current != mc.moving_task) { 1474 if (mem_cgroup_under_move(memcg)) { 1475 DEFINE_WAIT(wait); 1476 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE); 1477 /* moving charge context might have finished. */ 1478 if (mc.moving_task) 1479 schedule(); 1480 finish_wait(&mc.waitq, &wait); 1481 return true; 1482 } 1483 } 1484 return false; 1485 } 1486 1487 struct memory_stat { 1488 const char *name; 1489 unsigned int idx; 1490 }; 1491 1492 static const struct memory_stat memory_stats[] = { 1493 { "anon", NR_ANON_MAPPED }, 1494 { "file", NR_FILE_PAGES }, 1495 { "kernel", MEMCG_KMEM }, 1496 { "kernel_stack", NR_KERNEL_STACK_KB }, 1497 { "pagetables", NR_PAGETABLE }, 1498 { "sec_pagetables", NR_SECONDARY_PAGETABLE }, 1499 { "percpu", MEMCG_PERCPU_B }, 1500 { "sock", MEMCG_SOCK }, 1501 { "vmalloc", MEMCG_VMALLOC }, 1502 { "shmem", NR_SHMEM }, 1503 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP) 1504 { "zswap", MEMCG_ZSWAP_B }, 1505 { "zswapped", MEMCG_ZSWAPPED }, 1506 #endif 1507 { "file_mapped", NR_FILE_MAPPED }, 1508 { "file_dirty", NR_FILE_DIRTY }, 1509 { "file_writeback", NR_WRITEBACK }, 1510 #ifdef CONFIG_SWAP 1511 { "swapcached", NR_SWAPCACHE }, 1512 #endif 1513 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1514 { "anon_thp", NR_ANON_THPS }, 1515 { "file_thp", NR_FILE_THPS }, 1516 { "shmem_thp", NR_SHMEM_THPS }, 1517 #endif 1518 { "inactive_anon", NR_INACTIVE_ANON }, 1519 { "active_anon", NR_ACTIVE_ANON }, 1520 { "inactive_file", NR_INACTIVE_FILE }, 1521 { "active_file", NR_ACTIVE_FILE }, 1522 { "unevictable", NR_UNEVICTABLE }, 1523 { "slab_reclaimable", NR_SLAB_RECLAIMABLE_B }, 1524 { "slab_unreclaimable", NR_SLAB_UNRECLAIMABLE_B }, 1525 1526 /* The memory events */ 1527 { "workingset_refault_anon", WORKINGSET_REFAULT_ANON }, 1528 { "workingset_refault_file", WORKINGSET_REFAULT_FILE }, 1529 { "workingset_activate_anon", WORKINGSET_ACTIVATE_ANON }, 1530 { "workingset_activate_file", WORKINGSET_ACTIVATE_FILE }, 1531 { "workingset_restore_anon", WORKINGSET_RESTORE_ANON }, 1532 { "workingset_restore_file", WORKINGSET_RESTORE_FILE }, 1533 { "workingset_nodereclaim", WORKINGSET_NODERECLAIM }, 1534 }; 1535 1536 /* Translate stat items to the correct unit for memory.stat output */ 1537 static int memcg_page_state_unit(int item) 1538 { 1539 switch (item) { 1540 case MEMCG_PERCPU_B: 1541 case MEMCG_ZSWAP_B: 1542 case NR_SLAB_RECLAIMABLE_B: 1543 case NR_SLAB_UNRECLAIMABLE_B: 1544 case WORKINGSET_REFAULT_ANON: 1545 case WORKINGSET_REFAULT_FILE: 1546 case WORKINGSET_ACTIVATE_ANON: 1547 case WORKINGSET_ACTIVATE_FILE: 1548 case WORKINGSET_RESTORE_ANON: 1549 case WORKINGSET_RESTORE_FILE: 1550 case WORKINGSET_NODERECLAIM: 1551 return 1; 1552 case NR_KERNEL_STACK_KB: 1553 return SZ_1K; 1554 default: 1555 return PAGE_SIZE; 1556 } 1557 } 1558 1559 static inline unsigned long memcg_page_state_output(struct mem_cgroup *memcg, 1560 int item) 1561 { 1562 return memcg_page_state(memcg, item) * memcg_page_state_unit(item); 1563 } 1564 1565 static void memcg_stat_format(struct mem_cgroup *memcg, struct seq_buf *s) 1566 { 1567 int i; 1568 1569 /* 1570 * Provide statistics on the state of the memory subsystem as 1571 * well as cumulative event counters that show past behavior. 1572 * 1573 * This list is ordered following a combination of these gradients: 1574 * 1) generic big picture -> specifics and details 1575 * 2) reflecting userspace activity -> reflecting kernel heuristics 1576 * 1577 * Current memory state: 1578 */ 1579 mem_cgroup_flush_stats(); 1580 1581 for (i = 0; i < ARRAY_SIZE(memory_stats); i++) { 1582 u64 size; 1583 1584 size = memcg_page_state_output(memcg, memory_stats[i].idx); 1585 seq_buf_printf(s, "%s %llu\n", memory_stats[i].name, size); 1586 1587 if (unlikely(memory_stats[i].idx == NR_SLAB_UNRECLAIMABLE_B)) { 1588 size += memcg_page_state_output(memcg, 1589 NR_SLAB_RECLAIMABLE_B); 1590 seq_buf_printf(s, "slab %llu\n", size); 1591 } 1592 } 1593 1594 /* Accumulated memory events */ 1595 seq_buf_printf(s, "pgscan %lu\n", 1596 memcg_events(memcg, PGSCAN_KSWAPD) + 1597 memcg_events(memcg, PGSCAN_DIRECT) + 1598 memcg_events(memcg, PGSCAN_KHUGEPAGED)); 1599 seq_buf_printf(s, "pgsteal %lu\n", 1600 memcg_events(memcg, PGSTEAL_KSWAPD) + 1601 memcg_events(memcg, PGSTEAL_DIRECT) + 1602 memcg_events(memcg, PGSTEAL_KHUGEPAGED)); 1603 1604 for (i = 0; i < ARRAY_SIZE(memcg_vm_event_stat); i++) { 1605 if (memcg_vm_event_stat[i] == PGPGIN || 1606 memcg_vm_event_stat[i] == PGPGOUT) 1607 continue; 1608 1609 seq_buf_printf(s, "%s %lu\n", 1610 vm_event_name(memcg_vm_event_stat[i]), 1611 memcg_events(memcg, memcg_vm_event_stat[i])); 1612 } 1613 1614 /* The above should easily fit into one page */ 1615 WARN_ON_ONCE(seq_buf_has_overflowed(s)); 1616 } 1617 1618 static void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s); 1619 1620 static void memory_stat_format(struct mem_cgroup *memcg, struct seq_buf *s) 1621 { 1622 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 1623 memcg_stat_format(memcg, s); 1624 else 1625 memcg1_stat_format(memcg, s); 1626 WARN_ON_ONCE(seq_buf_has_overflowed(s)); 1627 } 1628 1629 /** 1630 * mem_cgroup_print_oom_context: Print OOM information relevant to 1631 * memory controller. 1632 * @memcg: The memory cgroup that went over limit 1633 * @p: Task that is going to be killed 1634 * 1635 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is 1636 * enabled 1637 */ 1638 void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p) 1639 { 1640 rcu_read_lock(); 1641 1642 if (memcg) { 1643 pr_cont(",oom_memcg="); 1644 pr_cont_cgroup_path(memcg->css.cgroup); 1645 } else 1646 pr_cont(",global_oom"); 1647 if (p) { 1648 pr_cont(",task_memcg="); 1649 pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id)); 1650 } 1651 rcu_read_unlock(); 1652 } 1653 1654 /** 1655 * mem_cgroup_print_oom_meminfo: Print OOM memory information relevant to 1656 * memory controller. 1657 * @memcg: The memory cgroup that went over limit 1658 */ 1659 void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg) 1660 { 1661 /* Use static buffer, for the caller is holding oom_lock. */ 1662 static char buf[PAGE_SIZE]; 1663 struct seq_buf s; 1664 1665 lockdep_assert_held(&oom_lock); 1666 1667 pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n", 1668 K((u64)page_counter_read(&memcg->memory)), 1669 K((u64)READ_ONCE(memcg->memory.max)), memcg->memory.failcnt); 1670 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 1671 pr_info("swap: usage %llukB, limit %llukB, failcnt %lu\n", 1672 K((u64)page_counter_read(&memcg->swap)), 1673 K((u64)READ_ONCE(memcg->swap.max)), memcg->swap.failcnt); 1674 else { 1675 pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n", 1676 K((u64)page_counter_read(&memcg->memsw)), 1677 K((u64)memcg->memsw.max), memcg->memsw.failcnt); 1678 pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n", 1679 K((u64)page_counter_read(&memcg->kmem)), 1680 K((u64)memcg->kmem.max), memcg->kmem.failcnt); 1681 } 1682 1683 pr_info("Memory cgroup stats for "); 1684 pr_cont_cgroup_path(memcg->css.cgroup); 1685 pr_cont(":"); 1686 seq_buf_init(&s, buf, sizeof(buf)); 1687 memory_stat_format(memcg, &s); 1688 seq_buf_do_printk(&s, KERN_INFO); 1689 } 1690 1691 /* 1692 * Return the memory (and swap, if configured) limit for a memcg. 1693 */ 1694 unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg) 1695 { 1696 unsigned long max = READ_ONCE(memcg->memory.max); 1697 1698 if (do_memsw_account()) { 1699 if (mem_cgroup_swappiness(memcg)) { 1700 /* Calculate swap excess capacity from memsw limit */ 1701 unsigned long swap = READ_ONCE(memcg->memsw.max) - max; 1702 1703 max += min(swap, (unsigned long)total_swap_pages); 1704 } 1705 } else { 1706 if (mem_cgroup_swappiness(memcg)) 1707 max += min(READ_ONCE(memcg->swap.max), 1708 (unsigned long)total_swap_pages); 1709 } 1710 return max; 1711 } 1712 1713 unsigned long mem_cgroup_size(struct mem_cgroup *memcg) 1714 { 1715 return page_counter_read(&memcg->memory); 1716 } 1717 1718 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask, 1719 int order) 1720 { 1721 struct oom_control oc = { 1722 .zonelist = NULL, 1723 .nodemask = NULL, 1724 .memcg = memcg, 1725 .gfp_mask = gfp_mask, 1726 .order = order, 1727 }; 1728 bool ret = true; 1729 1730 if (mutex_lock_killable(&oom_lock)) 1731 return true; 1732 1733 if (mem_cgroup_margin(memcg) >= (1 << order)) 1734 goto unlock; 1735 1736 /* 1737 * A few threads which were not waiting at mutex_lock_killable() can 1738 * fail to bail out. Therefore, check again after holding oom_lock. 1739 */ 1740 ret = task_is_dying() || out_of_memory(&oc); 1741 1742 unlock: 1743 mutex_unlock(&oom_lock); 1744 return ret; 1745 } 1746 1747 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg, 1748 pg_data_t *pgdat, 1749 gfp_t gfp_mask, 1750 unsigned long *total_scanned) 1751 { 1752 struct mem_cgroup *victim = NULL; 1753 int total = 0; 1754 int loop = 0; 1755 unsigned long excess; 1756 unsigned long nr_scanned; 1757 struct mem_cgroup_reclaim_cookie reclaim = { 1758 .pgdat = pgdat, 1759 }; 1760 1761 excess = soft_limit_excess(root_memcg); 1762 1763 while (1) { 1764 victim = mem_cgroup_iter(root_memcg, victim, &reclaim); 1765 if (!victim) { 1766 loop++; 1767 if (loop >= 2) { 1768 /* 1769 * If we have not been able to reclaim 1770 * anything, it might because there are 1771 * no reclaimable pages under this hierarchy 1772 */ 1773 if (!total) 1774 break; 1775 /* 1776 * We want to do more targeted reclaim. 1777 * excess >> 2 is not to excessive so as to 1778 * reclaim too much, nor too less that we keep 1779 * coming back to reclaim from this cgroup 1780 */ 1781 if (total >= (excess >> 2) || 1782 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) 1783 break; 1784 } 1785 continue; 1786 } 1787 total += mem_cgroup_shrink_node(victim, gfp_mask, false, 1788 pgdat, &nr_scanned); 1789 *total_scanned += nr_scanned; 1790 if (!soft_limit_excess(root_memcg)) 1791 break; 1792 } 1793 mem_cgroup_iter_break(root_memcg, victim); 1794 return total; 1795 } 1796 1797 #ifdef CONFIG_LOCKDEP 1798 static struct lockdep_map memcg_oom_lock_dep_map = { 1799 .name = "memcg_oom_lock", 1800 }; 1801 #endif 1802 1803 static DEFINE_SPINLOCK(memcg_oom_lock); 1804 1805 /* 1806 * Check OOM-Killer is already running under our hierarchy. 1807 * If someone is running, return false. 1808 */ 1809 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg) 1810 { 1811 struct mem_cgroup *iter, *failed = NULL; 1812 1813 spin_lock(&memcg_oom_lock); 1814 1815 for_each_mem_cgroup_tree(iter, memcg) { 1816 if (iter->oom_lock) { 1817 /* 1818 * this subtree of our hierarchy is already locked 1819 * so we cannot give a lock. 1820 */ 1821 failed = iter; 1822 mem_cgroup_iter_break(memcg, iter); 1823 break; 1824 } else 1825 iter->oom_lock = true; 1826 } 1827 1828 if (failed) { 1829 /* 1830 * OK, we failed to lock the whole subtree so we have 1831 * to clean up what we set up to the failing subtree 1832 */ 1833 for_each_mem_cgroup_tree(iter, memcg) { 1834 if (iter == failed) { 1835 mem_cgroup_iter_break(memcg, iter); 1836 break; 1837 } 1838 iter->oom_lock = false; 1839 } 1840 } else 1841 mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_); 1842 1843 spin_unlock(&memcg_oom_lock); 1844 1845 return !failed; 1846 } 1847 1848 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg) 1849 { 1850 struct mem_cgroup *iter; 1851 1852 spin_lock(&memcg_oom_lock); 1853 mutex_release(&memcg_oom_lock_dep_map, _RET_IP_); 1854 for_each_mem_cgroup_tree(iter, memcg) 1855 iter->oom_lock = false; 1856 spin_unlock(&memcg_oom_lock); 1857 } 1858 1859 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg) 1860 { 1861 struct mem_cgroup *iter; 1862 1863 spin_lock(&memcg_oom_lock); 1864 for_each_mem_cgroup_tree(iter, memcg) 1865 iter->under_oom++; 1866 spin_unlock(&memcg_oom_lock); 1867 } 1868 1869 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg) 1870 { 1871 struct mem_cgroup *iter; 1872 1873 /* 1874 * Be careful about under_oom underflows because a child memcg 1875 * could have been added after mem_cgroup_mark_under_oom. 1876 */ 1877 spin_lock(&memcg_oom_lock); 1878 for_each_mem_cgroup_tree(iter, memcg) 1879 if (iter->under_oom > 0) 1880 iter->under_oom--; 1881 spin_unlock(&memcg_oom_lock); 1882 } 1883 1884 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq); 1885 1886 struct oom_wait_info { 1887 struct mem_cgroup *memcg; 1888 wait_queue_entry_t wait; 1889 }; 1890 1891 static int memcg_oom_wake_function(wait_queue_entry_t *wait, 1892 unsigned mode, int sync, void *arg) 1893 { 1894 struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg; 1895 struct mem_cgroup *oom_wait_memcg; 1896 struct oom_wait_info *oom_wait_info; 1897 1898 oom_wait_info = container_of(wait, struct oom_wait_info, wait); 1899 oom_wait_memcg = oom_wait_info->memcg; 1900 1901 if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) && 1902 !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg)) 1903 return 0; 1904 return autoremove_wake_function(wait, mode, sync, arg); 1905 } 1906 1907 static void memcg_oom_recover(struct mem_cgroup *memcg) 1908 { 1909 /* 1910 * For the following lockless ->under_oom test, the only required 1911 * guarantee is that it must see the state asserted by an OOM when 1912 * this function is called as a result of userland actions 1913 * triggered by the notification of the OOM. This is trivially 1914 * achieved by invoking mem_cgroup_mark_under_oom() before 1915 * triggering notification. 1916 */ 1917 if (memcg && memcg->under_oom) 1918 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg); 1919 } 1920 1921 /* 1922 * Returns true if successfully killed one or more processes. Though in some 1923 * corner cases it can return true even without killing any process. 1924 */ 1925 static bool mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order) 1926 { 1927 bool locked, ret; 1928 1929 if (order > PAGE_ALLOC_COSTLY_ORDER) 1930 return false; 1931 1932 memcg_memory_event(memcg, MEMCG_OOM); 1933 1934 /* 1935 * We are in the middle of the charge context here, so we 1936 * don't want to block when potentially sitting on a callstack 1937 * that holds all kinds of filesystem and mm locks. 1938 * 1939 * cgroup1 allows disabling the OOM killer and waiting for outside 1940 * handling until the charge can succeed; remember the context and put 1941 * the task to sleep at the end of the page fault when all locks are 1942 * released. 1943 * 1944 * On the other hand, in-kernel OOM killer allows for an async victim 1945 * memory reclaim (oom_reaper) and that means that we are not solely 1946 * relying on the oom victim to make a forward progress and we can 1947 * invoke the oom killer here. 1948 * 1949 * Please note that mem_cgroup_out_of_memory might fail to find a 1950 * victim and then we have to bail out from the charge path. 1951 */ 1952 if (READ_ONCE(memcg->oom_kill_disable)) { 1953 if (current->in_user_fault) { 1954 css_get(&memcg->css); 1955 current->memcg_in_oom = memcg; 1956 current->memcg_oom_gfp_mask = mask; 1957 current->memcg_oom_order = order; 1958 } 1959 return false; 1960 } 1961 1962 mem_cgroup_mark_under_oom(memcg); 1963 1964 locked = mem_cgroup_oom_trylock(memcg); 1965 1966 if (locked) 1967 mem_cgroup_oom_notify(memcg); 1968 1969 mem_cgroup_unmark_under_oom(memcg); 1970 ret = mem_cgroup_out_of_memory(memcg, mask, order); 1971 1972 if (locked) 1973 mem_cgroup_oom_unlock(memcg); 1974 1975 return ret; 1976 } 1977 1978 /** 1979 * mem_cgroup_oom_synchronize - complete memcg OOM handling 1980 * @handle: actually kill/wait or just clean up the OOM state 1981 * 1982 * This has to be called at the end of a page fault if the memcg OOM 1983 * handler was enabled. 1984 * 1985 * Memcg supports userspace OOM handling where failed allocations must 1986 * sleep on a waitqueue until the userspace task resolves the 1987 * situation. Sleeping directly in the charge context with all kinds 1988 * of locks held is not a good idea, instead we remember an OOM state 1989 * in the task and mem_cgroup_oom_synchronize() has to be called at 1990 * the end of the page fault to complete the OOM handling. 1991 * 1992 * Returns %true if an ongoing memcg OOM situation was detected and 1993 * completed, %false otherwise. 1994 */ 1995 bool mem_cgroup_oom_synchronize(bool handle) 1996 { 1997 struct mem_cgroup *memcg = current->memcg_in_oom; 1998 struct oom_wait_info owait; 1999 bool locked; 2000 2001 /* OOM is global, do not handle */ 2002 if (!memcg) 2003 return false; 2004 2005 if (!handle) 2006 goto cleanup; 2007 2008 owait.memcg = memcg; 2009 owait.wait.flags = 0; 2010 owait.wait.func = memcg_oom_wake_function; 2011 owait.wait.private = current; 2012 INIT_LIST_HEAD(&owait.wait.entry); 2013 2014 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE); 2015 mem_cgroup_mark_under_oom(memcg); 2016 2017 locked = mem_cgroup_oom_trylock(memcg); 2018 2019 if (locked) 2020 mem_cgroup_oom_notify(memcg); 2021 2022 schedule(); 2023 mem_cgroup_unmark_under_oom(memcg); 2024 finish_wait(&memcg_oom_waitq, &owait.wait); 2025 2026 if (locked) 2027 mem_cgroup_oom_unlock(memcg); 2028 cleanup: 2029 current->memcg_in_oom = NULL; 2030 css_put(&memcg->css); 2031 return true; 2032 } 2033 2034 /** 2035 * mem_cgroup_get_oom_group - get a memory cgroup to clean up after OOM 2036 * @victim: task to be killed by the OOM killer 2037 * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM 2038 * 2039 * Returns a pointer to a memory cgroup, which has to be cleaned up 2040 * by killing all belonging OOM-killable tasks. 2041 * 2042 * Caller has to call mem_cgroup_put() on the returned non-NULL memcg. 2043 */ 2044 struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim, 2045 struct mem_cgroup *oom_domain) 2046 { 2047 struct mem_cgroup *oom_group = NULL; 2048 struct mem_cgroup *memcg; 2049 2050 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) 2051 return NULL; 2052 2053 if (!oom_domain) 2054 oom_domain = root_mem_cgroup; 2055 2056 rcu_read_lock(); 2057 2058 memcg = mem_cgroup_from_task(victim); 2059 if (mem_cgroup_is_root(memcg)) 2060 goto out; 2061 2062 /* 2063 * If the victim task has been asynchronously moved to a different 2064 * memory cgroup, we might end up killing tasks outside oom_domain. 2065 * In this case it's better to ignore memory.group.oom. 2066 */ 2067 if (unlikely(!mem_cgroup_is_descendant(memcg, oom_domain))) 2068 goto out; 2069 2070 /* 2071 * Traverse the memory cgroup hierarchy from the victim task's 2072 * cgroup up to the OOMing cgroup (or root) to find the 2073 * highest-level memory cgroup with oom.group set. 2074 */ 2075 for (; memcg; memcg = parent_mem_cgroup(memcg)) { 2076 if (READ_ONCE(memcg->oom_group)) 2077 oom_group = memcg; 2078 2079 if (memcg == oom_domain) 2080 break; 2081 } 2082 2083 if (oom_group) 2084 css_get(&oom_group->css); 2085 out: 2086 rcu_read_unlock(); 2087 2088 return oom_group; 2089 } 2090 2091 void mem_cgroup_print_oom_group(struct mem_cgroup *memcg) 2092 { 2093 pr_info("Tasks in "); 2094 pr_cont_cgroup_path(memcg->css.cgroup); 2095 pr_cont(" are going to be killed due to memory.oom.group set\n"); 2096 } 2097 2098 /** 2099 * folio_memcg_lock - Bind a folio to its memcg. 2100 * @folio: The folio. 2101 * 2102 * This function prevents unlocked LRU folios from being moved to 2103 * another cgroup. 2104 * 2105 * It ensures lifetime of the bound memcg. The caller is responsible 2106 * for the lifetime of the folio. 2107 */ 2108 void folio_memcg_lock(struct folio *folio) 2109 { 2110 struct mem_cgroup *memcg; 2111 unsigned long flags; 2112 2113 /* 2114 * The RCU lock is held throughout the transaction. The fast 2115 * path can get away without acquiring the memcg->move_lock 2116 * because page moving starts with an RCU grace period. 2117 */ 2118 rcu_read_lock(); 2119 2120 if (mem_cgroup_disabled()) 2121 return; 2122 again: 2123 memcg = folio_memcg(folio); 2124 if (unlikely(!memcg)) 2125 return; 2126 2127 #ifdef CONFIG_PROVE_LOCKING 2128 local_irq_save(flags); 2129 might_lock(&memcg->move_lock); 2130 local_irq_restore(flags); 2131 #endif 2132 2133 if (atomic_read(&memcg->moving_account) <= 0) 2134 return; 2135 2136 spin_lock_irqsave(&memcg->move_lock, flags); 2137 if (memcg != folio_memcg(folio)) { 2138 spin_unlock_irqrestore(&memcg->move_lock, flags); 2139 goto again; 2140 } 2141 2142 /* 2143 * When charge migration first begins, we can have multiple 2144 * critical sections holding the fast-path RCU lock and one 2145 * holding the slowpath move_lock. Track the task who has the 2146 * move_lock for folio_memcg_unlock(). 2147 */ 2148 memcg->move_lock_task = current; 2149 memcg->move_lock_flags = flags; 2150 } 2151 2152 static void __folio_memcg_unlock(struct mem_cgroup *memcg) 2153 { 2154 if (memcg && memcg->move_lock_task == current) { 2155 unsigned long flags = memcg->move_lock_flags; 2156 2157 memcg->move_lock_task = NULL; 2158 memcg->move_lock_flags = 0; 2159 2160 spin_unlock_irqrestore(&memcg->move_lock, flags); 2161 } 2162 2163 rcu_read_unlock(); 2164 } 2165 2166 /** 2167 * folio_memcg_unlock - Release the binding between a folio and its memcg. 2168 * @folio: The folio. 2169 * 2170 * This releases the binding created by folio_memcg_lock(). This does 2171 * not change the accounting of this folio to its memcg, but it does 2172 * permit others to change it. 2173 */ 2174 void folio_memcg_unlock(struct folio *folio) 2175 { 2176 __folio_memcg_unlock(folio_memcg(folio)); 2177 } 2178 2179 struct memcg_stock_pcp { 2180 local_lock_t stock_lock; 2181 struct mem_cgroup *cached; /* this never be root cgroup */ 2182 unsigned int nr_pages; 2183 2184 #ifdef CONFIG_MEMCG_KMEM 2185 struct obj_cgroup *cached_objcg; 2186 struct pglist_data *cached_pgdat; 2187 unsigned int nr_bytes; 2188 int nr_slab_reclaimable_b; 2189 int nr_slab_unreclaimable_b; 2190 #endif 2191 2192 struct work_struct work; 2193 unsigned long flags; 2194 #define FLUSHING_CACHED_CHARGE 0 2195 }; 2196 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock) = { 2197 .stock_lock = INIT_LOCAL_LOCK(stock_lock), 2198 }; 2199 static DEFINE_MUTEX(percpu_charge_mutex); 2200 2201 #ifdef CONFIG_MEMCG_KMEM 2202 static struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock); 2203 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock, 2204 struct mem_cgroup *root_memcg); 2205 static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages); 2206 2207 #else 2208 static inline struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock) 2209 { 2210 return NULL; 2211 } 2212 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock, 2213 struct mem_cgroup *root_memcg) 2214 { 2215 return false; 2216 } 2217 static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages) 2218 { 2219 } 2220 #endif 2221 2222 /** 2223 * consume_stock: Try to consume stocked charge on this cpu. 2224 * @memcg: memcg to consume from. 2225 * @nr_pages: how many pages to charge. 2226 * 2227 * The charges will only happen if @memcg matches the current cpu's memcg 2228 * stock, and at least @nr_pages are available in that stock. Failure to 2229 * service an allocation will refill the stock. 2230 * 2231 * returns true if successful, false otherwise. 2232 */ 2233 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) 2234 { 2235 struct memcg_stock_pcp *stock; 2236 unsigned long flags; 2237 bool ret = false; 2238 2239 if (nr_pages > MEMCG_CHARGE_BATCH) 2240 return ret; 2241 2242 local_lock_irqsave(&memcg_stock.stock_lock, flags); 2243 2244 stock = this_cpu_ptr(&memcg_stock); 2245 if (memcg == READ_ONCE(stock->cached) && stock->nr_pages >= nr_pages) { 2246 stock->nr_pages -= nr_pages; 2247 ret = true; 2248 } 2249 2250 local_unlock_irqrestore(&memcg_stock.stock_lock, flags); 2251 2252 return ret; 2253 } 2254 2255 /* 2256 * Returns stocks cached in percpu and reset cached information. 2257 */ 2258 static void drain_stock(struct memcg_stock_pcp *stock) 2259 { 2260 struct mem_cgroup *old = READ_ONCE(stock->cached); 2261 2262 if (!old) 2263 return; 2264 2265 if (stock->nr_pages) { 2266 page_counter_uncharge(&old->memory, stock->nr_pages); 2267 if (do_memsw_account()) 2268 page_counter_uncharge(&old->memsw, stock->nr_pages); 2269 stock->nr_pages = 0; 2270 } 2271 2272 css_put(&old->css); 2273 WRITE_ONCE(stock->cached, NULL); 2274 } 2275 2276 static void drain_local_stock(struct work_struct *dummy) 2277 { 2278 struct memcg_stock_pcp *stock; 2279 struct obj_cgroup *old = NULL; 2280 unsigned long flags; 2281 2282 /* 2283 * The only protection from cpu hotplug (memcg_hotplug_cpu_dead) vs. 2284 * drain_stock races is that we always operate on local CPU stock 2285 * here with IRQ disabled 2286 */ 2287 local_lock_irqsave(&memcg_stock.stock_lock, flags); 2288 2289 stock = this_cpu_ptr(&memcg_stock); 2290 old = drain_obj_stock(stock); 2291 drain_stock(stock); 2292 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags); 2293 2294 local_unlock_irqrestore(&memcg_stock.stock_lock, flags); 2295 if (old) 2296 obj_cgroup_put(old); 2297 } 2298 2299 /* 2300 * Cache charges(val) to local per_cpu area. 2301 * This will be consumed by consume_stock() function, later. 2302 */ 2303 static void __refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) 2304 { 2305 struct memcg_stock_pcp *stock; 2306 2307 stock = this_cpu_ptr(&memcg_stock); 2308 if (READ_ONCE(stock->cached) != memcg) { /* reset if necessary */ 2309 drain_stock(stock); 2310 css_get(&memcg->css); 2311 WRITE_ONCE(stock->cached, memcg); 2312 } 2313 stock->nr_pages += nr_pages; 2314 2315 if (stock->nr_pages > MEMCG_CHARGE_BATCH) 2316 drain_stock(stock); 2317 } 2318 2319 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) 2320 { 2321 unsigned long flags; 2322 2323 local_lock_irqsave(&memcg_stock.stock_lock, flags); 2324 __refill_stock(memcg, nr_pages); 2325 local_unlock_irqrestore(&memcg_stock.stock_lock, flags); 2326 } 2327 2328 /* 2329 * Drains all per-CPU charge caches for given root_memcg resp. subtree 2330 * of the hierarchy under it. 2331 */ 2332 static void drain_all_stock(struct mem_cgroup *root_memcg) 2333 { 2334 int cpu, curcpu; 2335 2336 /* If someone's already draining, avoid adding running more workers. */ 2337 if (!mutex_trylock(&percpu_charge_mutex)) 2338 return; 2339 /* 2340 * Notify other cpus that system-wide "drain" is running 2341 * We do not care about races with the cpu hotplug because cpu down 2342 * as well as workers from this path always operate on the local 2343 * per-cpu data. CPU up doesn't touch memcg_stock at all. 2344 */ 2345 migrate_disable(); 2346 curcpu = smp_processor_id(); 2347 for_each_online_cpu(cpu) { 2348 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); 2349 struct mem_cgroup *memcg; 2350 bool flush = false; 2351 2352 rcu_read_lock(); 2353 memcg = READ_ONCE(stock->cached); 2354 if (memcg && stock->nr_pages && 2355 mem_cgroup_is_descendant(memcg, root_memcg)) 2356 flush = true; 2357 else if (obj_stock_flush_required(stock, root_memcg)) 2358 flush = true; 2359 rcu_read_unlock(); 2360 2361 if (flush && 2362 !test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) { 2363 if (cpu == curcpu) 2364 drain_local_stock(&stock->work); 2365 else if (!cpu_is_isolated(cpu)) 2366 schedule_work_on(cpu, &stock->work); 2367 } 2368 } 2369 migrate_enable(); 2370 mutex_unlock(&percpu_charge_mutex); 2371 } 2372 2373 static int memcg_hotplug_cpu_dead(unsigned int cpu) 2374 { 2375 struct memcg_stock_pcp *stock; 2376 2377 stock = &per_cpu(memcg_stock, cpu); 2378 drain_stock(stock); 2379 2380 return 0; 2381 } 2382 2383 static unsigned long reclaim_high(struct mem_cgroup *memcg, 2384 unsigned int nr_pages, 2385 gfp_t gfp_mask) 2386 { 2387 unsigned long nr_reclaimed = 0; 2388 2389 do { 2390 unsigned long pflags; 2391 2392 if (page_counter_read(&memcg->memory) <= 2393 READ_ONCE(memcg->memory.high)) 2394 continue; 2395 2396 memcg_memory_event(memcg, MEMCG_HIGH); 2397 2398 psi_memstall_enter(&pflags); 2399 nr_reclaimed += try_to_free_mem_cgroup_pages(memcg, nr_pages, 2400 gfp_mask, 2401 MEMCG_RECLAIM_MAY_SWAP); 2402 psi_memstall_leave(&pflags); 2403 } while ((memcg = parent_mem_cgroup(memcg)) && 2404 !mem_cgroup_is_root(memcg)); 2405 2406 return nr_reclaimed; 2407 } 2408 2409 static void high_work_func(struct work_struct *work) 2410 { 2411 struct mem_cgroup *memcg; 2412 2413 memcg = container_of(work, struct mem_cgroup, high_work); 2414 reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL); 2415 } 2416 2417 /* 2418 * Clamp the maximum sleep time per allocation batch to 2 seconds. This is 2419 * enough to still cause a significant slowdown in most cases, while still 2420 * allowing diagnostics and tracing to proceed without becoming stuck. 2421 */ 2422 #define MEMCG_MAX_HIGH_DELAY_JIFFIES (2UL*HZ) 2423 2424 /* 2425 * When calculating the delay, we use these either side of the exponentiation to 2426 * maintain precision and scale to a reasonable number of jiffies (see the table 2427 * below. 2428 * 2429 * - MEMCG_DELAY_PRECISION_SHIFT: Extra precision bits while translating the 2430 * overage ratio to a delay. 2431 * - MEMCG_DELAY_SCALING_SHIFT: The number of bits to scale down the 2432 * proposed penalty in order to reduce to a reasonable number of jiffies, and 2433 * to produce a reasonable delay curve. 2434 * 2435 * MEMCG_DELAY_SCALING_SHIFT just happens to be a number that produces a 2436 * reasonable delay curve compared to precision-adjusted overage, not 2437 * penalising heavily at first, but still making sure that growth beyond the 2438 * limit penalises misbehaviour cgroups by slowing them down exponentially. For 2439 * example, with a high of 100 megabytes: 2440 * 2441 * +-------+------------------------+ 2442 * | usage | time to allocate in ms | 2443 * +-------+------------------------+ 2444 * | 100M | 0 | 2445 * | 101M | 6 | 2446 * | 102M | 25 | 2447 * | 103M | 57 | 2448 * | 104M | 102 | 2449 * | 105M | 159 | 2450 * | 106M | 230 | 2451 * | 107M | 313 | 2452 * | 108M | 409 | 2453 * | 109M | 518 | 2454 * | 110M | 639 | 2455 * | 111M | 774 | 2456 * | 112M | 921 | 2457 * | 113M | 1081 | 2458 * | 114M | 1254 | 2459 * | 115M | 1439 | 2460 * | 116M | 1638 | 2461 * | 117M | 1849 | 2462 * | 118M | 2000 | 2463 * | 119M | 2000 | 2464 * | 120M | 2000 | 2465 * +-------+------------------------+ 2466 */ 2467 #define MEMCG_DELAY_PRECISION_SHIFT 20 2468 #define MEMCG_DELAY_SCALING_SHIFT 14 2469 2470 static u64 calculate_overage(unsigned long usage, unsigned long high) 2471 { 2472 u64 overage; 2473 2474 if (usage <= high) 2475 return 0; 2476 2477 /* 2478 * Prevent division by 0 in overage calculation by acting as if 2479 * it was a threshold of 1 page 2480 */ 2481 high = max(high, 1UL); 2482 2483 overage = usage - high; 2484 overage <<= MEMCG_DELAY_PRECISION_SHIFT; 2485 return div64_u64(overage, high); 2486 } 2487 2488 static u64 mem_find_max_overage(struct mem_cgroup *memcg) 2489 { 2490 u64 overage, max_overage = 0; 2491 2492 do { 2493 overage = calculate_overage(page_counter_read(&memcg->memory), 2494 READ_ONCE(memcg->memory.high)); 2495 max_overage = max(overage, max_overage); 2496 } while ((memcg = parent_mem_cgroup(memcg)) && 2497 !mem_cgroup_is_root(memcg)); 2498 2499 return max_overage; 2500 } 2501 2502 static u64 swap_find_max_overage(struct mem_cgroup *memcg) 2503 { 2504 u64 overage, max_overage = 0; 2505 2506 do { 2507 overage = calculate_overage(page_counter_read(&memcg->swap), 2508 READ_ONCE(memcg->swap.high)); 2509 if (overage) 2510 memcg_memory_event(memcg, MEMCG_SWAP_HIGH); 2511 max_overage = max(overage, max_overage); 2512 } while ((memcg = parent_mem_cgroup(memcg)) && 2513 !mem_cgroup_is_root(memcg)); 2514 2515 return max_overage; 2516 } 2517 2518 /* 2519 * Get the number of jiffies that we should penalise a mischievous cgroup which 2520 * is exceeding its memory.high by checking both it and its ancestors. 2521 */ 2522 static unsigned long calculate_high_delay(struct mem_cgroup *memcg, 2523 unsigned int nr_pages, 2524 u64 max_overage) 2525 { 2526 unsigned long penalty_jiffies; 2527 2528 if (!max_overage) 2529 return 0; 2530 2531 /* 2532 * We use overage compared to memory.high to calculate the number of 2533 * jiffies to sleep (penalty_jiffies). Ideally this value should be 2534 * fairly lenient on small overages, and increasingly harsh when the 2535 * memcg in question makes it clear that it has no intention of stopping 2536 * its crazy behaviour, so we exponentially increase the delay based on 2537 * overage amount. 2538 */ 2539 penalty_jiffies = max_overage * max_overage * HZ; 2540 penalty_jiffies >>= MEMCG_DELAY_PRECISION_SHIFT; 2541 penalty_jiffies >>= MEMCG_DELAY_SCALING_SHIFT; 2542 2543 /* 2544 * Factor in the task's own contribution to the overage, such that four 2545 * N-sized allocations are throttled approximately the same as one 2546 * 4N-sized allocation. 2547 * 2548 * MEMCG_CHARGE_BATCH pages is nominal, so work out how much smaller or 2549 * larger the current charge patch is than that. 2550 */ 2551 return penalty_jiffies * nr_pages / MEMCG_CHARGE_BATCH; 2552 } 2553 2554 /* 2555 * Scheduled by try_charge() to be executed from the userland return path 2556 * and reclaims memory over the high limit. 2557 */ 2558 void mem_cgroup_handle_over_high(gfp_t gfp_mask) 2559 { 2560 unsigned long penalty_jiffies; 2561 unsigned long pflags; 2562 unsigned long nr_reclaimed; 2563 unsigned int nr_pages = current->memcg_nr_pages_over_high; 2564 int nr_retries = MAX_RECLAIM_RETRIES; 2565 struct mem_cgroup *memcg; 2566 bool in_retry = false; 2567 2568 if (likely(!nr_pages)) 2569 return; 2570 2571 memcg = get_mem_cgroup_from_mm(current->mm); 2572 current->memcg_nr_pages_over_high = 0; 2573 2574 retry_reclaim: 2575 /* 2576 * The allocating task should reclaim at least the batch size, but for 2577 * subsequent retries we only want to do what's necessary to prevent oom 2578 * or breaching resource isolation. 2579 * 2580 * This is distinct from memory.max or page allocator behaviour because 2581 * memory.high is currently batched, whereas memory.max and the page 2582 * allocator run every time an allocation is made. 2583 */ 2584 nr_reclaimed = reclaim_high(memcg, 2585 in_retry ? SWAP_CLUSTER_MAX : nr_pages, 2586 gfp_mask); 2587 2588 /* 2589 * memory.high is breached and reclaim is unable to keep up. Throttle 2590 * allocators proactively to slow down excessive growth. 2591 */ 2592 penalty_jiffies = calculate_high_delay(memcg, nr_pages, 2593 mem_find_max_overage(memcg)); 2594 2595 penalty_jiffies += calculate_high_delay(memcg, nr_pages, 2596 swap_find_max_overage(memcg)); 2597 2598 /* 2599 * Clamp the max delay per usermode return so as to still keep the 2600 * application moving forwards and also permit diagnostics, albeit 2601 * extremely slowly. 2602 */ 2603 penalty_jiffies = min(penalty_jiffies, MEMCG_MAX_HIGH_DELAY_JIFFIES); 2604 2605 /* 2606 * Don't sleep if the amount of jiffies this memcg owes us is so low 2607 * that it's not even worth doing, in an attempt to be nice to those who 2608 * go only a small amount over their memory.high value and maybe haven't 2609 * been aggressively reclaimed enough yet. 2610 */ 2611 if (penalty_jiffies <= HZ / 100) 2612 goto out; 2613 2614 /* 2615 * If reclaim is making forward progress but we're still over 2616 * memory.high, we want to encourage that rather than doing allocator 2617 * throttling. 2618 */ 2619 if (nr_reclaimed || nr_retries--) { 2620 in_retry = true; 2621 goto retry_reclaim; 2622 } 2623 2624 /* 2625 * If we exit early, we're guaranteed to die (since 2626 * schedule_timeout_killable sets TASK_KILLABLE). This means we don't 2627 * need to account for any ill-begotten jiffies to pay them off later. 2628 */ 2629 psi_memstall_enter(&pflags); 2630 schedule_timeout_killable(penalty_jiffies); 2631 psi_memstall_leave(&pflags); 2632 2633 out: 2634 css_put(&memcg->css); 2635 } 2636 2637 static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, 2638 unsigned int nr_pages) 2639 { 2640 unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages); 2641 int nr_retries = MAX_RECLAIM_RETRIES; 2642 struct mem_cgroup *mem_over_limit; 2643 struct page_counter *counter; 2644 unsigned long nr_reclaimed; 2645 bool passed_oom = false; 2646 unsigned int reclaim_options = MEMCG_RECLAIM_MAY_SWAP; 2647 bool drained = false; 2648 bool raised_max_event = false; 2649 unsigned long pflags; 2650 2651 retry: 2652 if (consume_stock(memcg, nr_pages)) 2653 return 0; 2654 2655 if (!do_memsw_account() || 2656 page_counter_try_charge(&memcg->memsw, batch, &counter)) { 2657 if (page_counter_try_charge(&memcg->memory, batch, &counter)) 2658 goto done_restock; 2659 if (do_memsw_account()) 2660 page_counter_uncharge(&memcg->memsw, batch); 2661 mem_over_limit = mem_cgroup_from_counter(counter, memory); 2662 } else { 2663 mem_over_limit = mem_cgroup_from_counter(counter, memsw); 2664 reclaim_options &= ~MEMCG_RECLAIM_MAY_SWAP; 2665 } 2666 2667 if (batch > nr_pages) { 2668 batch = nr_pages; 2669 goto retry; 2670 } 2671 2672 /* 2673 * Prevent unbounded recursion when reclaim operations need to 2674 * allocate memory. This might exceed the limits temporarily, 2675 * but we prefer facilitating memory reclaim and getting back 2676 * under the limit over triggering OOM kills in these cases. 2677 */ 2678 if (unlikely(current->flags & PF_MEMALLOC)) 2679 goto force; 2680 2681 if (unlikely(task_in_memcg_oom(current))) 2682 goto nomem; 2683 2684 if (!gfpflags_allow_blocking(gfp_mask)) 2685 goto nomem; 2686 2687 memcg_memory_event(mem_over_limit, MEMCG_MAX); 2688 raised_max_event = true; 2689 2690 psi_memstall_enter(&pflags); 2691 nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages, 2692 gfp_mask, reclaim_options); 2693 psi_memstall_leave(&pflags); 2694 2695 if (mem_cgroup_margin(mem_over_limit) >= nr_pages) 2696 goto retry; 2697 2698 if (!drained) { 2699 drain_all_stock(mem_over_limit); 2700 drained = true; 2701 goto retry; 2702 } 2703 2704 if (gfp_mask & __GFP_NORETRY) 2705 goto nomem; 2706 /* 2707 * Even though the limit is exceeded at this point, reclaim 2708 * may have been able to free some pages. Retry the charge 2709 * before killing the task. 2710 * 2711 * Only for regular pages, though: huge pages are rather 2712 * unlikely to succeed so close to the limit, and we fall back 2713 * to regular pages anyway in case of failure. 2714 */ 2715 if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER)) 2716 goto retry; 2717 /* 2718 * At task move, charge accounts can be doubly counted. So, it's 2719 * better to wait until the end of task_move if something is going on. 2720 */ 2721 if (mem_cgroup_wait_acct_move(mem_over_limit)) 2722 goto retry; 2723 2724 if (nr_retries--) 2725 goto retry; 2726 2727 if (gfp_mask & __GFP_RETRY_MAYFAIL) 2728 goto nomem; 2729 2730 /* Avoid endless loop for tasks bypassed by the oom killer */ 2731 if (passed_oom && task_is_dying()) 2732 goto nomem; 2733 2734 /* 2735 * keep retrying as long as the memcg oom killer is able to make 2736 * a forward progress or bypass the charge if the oom killer 2737 * couldn't make any progress. 2738 */ 2739 if (mem_cgroup_oom(mem_over_limit, gfp_mask, 2740 get_order(nr_pages * PAGE_SIZE))) { 2741 passed_oom = true; 2742 nr_retries = MAX_RECLAIM_RETRIES; 2743 goto retry; 2744 } 2745 nomem: 2746 /* 2747 * Memcg doesn't have a dedicated reserve for atomic 2748 * allocations. But like the global atomic pool, we need to 2749 * put the burden of reclaim on regular allocation requests 2750 * and let these go through as privileged allocations. 2751 */ 2752 if (!(gfp_mask & (__GFP_NOFAIL | __GFP_HIGH))) 2753 return -ENOMEM; 2754 force: 2755 /* 2756 * If the allocation has to be enforced, don't forget to raise 2757 * a MEMCG_MAX event. 2758 */ 2759 if (!raised_max_event) 2760 memcg_memory_event(mem_over_limit, MEMCG_MAX); 2761 2762 /* 2763 * The allocation either can't fail or will lead to more memory 2764 * being freed very soon. Allow memory usage go over the limit 2765 * temporarily by force charging it. 2766 */ 2767 page_counter_charge(&memcg->memory, nr_pages); 2768 if (do_memsw_account()) 2769 page_counter_charge(&memcg->memsw, nr_pages); 2770 2771 return 0; 2772 2773 done_restock: 2774 if (batch > nr_pages) 2775 refill_stock(memcg, batch - nr_pages); 2776 2777 /* 2778 * If the hierarchy is above the normal consumption range, schedule 2779 * reclaim on returning to userland. We can perform reclaim here 2780 * if __GFP_RECLAIM but let's always punt for simplicity and so that 2781 * GFP_KERNEL can consistently be used during reclaim. @memcg is 2782 * not recorded as it most likely matches current's and won't 2783 * change in the meantime. As high limit is checked again before 2784 * reclaim, the cost of mismatch is negligible. 2785 */ 2786 do { 2787 bool mem_high, swap_high; 2788 2789 mem_high = page_counter_read(&memcg->memory) > 2790 READ_ONCE(memcg->memory.high); 2791 swap_high = page_counter_read(&memcg->swap) > 2792 READ_ONCE(memcg->swap.high); 2793 2794 /* Don't bother a random interrupted task */ 2795 if (!in_task()) { 2796 if (mem_high) { 2797 schedule_work(&memcg->high_work); 2798 break; 2799 } 2800 continue; 2801 } 2802 2803 if (mem_high || swap_high) { 2804 /* 2805 * The allocating tasks in this cgroup will need to do 2806 * reclaim or be throttled to prevent further growth 2807 * of the memory or swap footprints. 2808 * 2809 * Target some best-effort fairness between the tasks, 2810 * and distribute reclaim work and delay penalties 2811 * based on how much each task is actually allocating. 2812 */ 2813 current->memcg_nr_pages_over_high += batch; 2814 set_notify_resume(current); 2815 break; 2816 } 2817 } while ((memcg = parent_mem_cgroup(memcg))); 2818 2819 if (current->memcg_nr_pages_over_high > MEMCG_CHARGE_BATCH && 2820 !(current->flags & PF_MEMALLOC) && 2821 gfpflags_allow_blocking(gfp_mask)) { 2822 mem_cgroup_handle_over_high(gfp_mask); 2823 } 2824 return 0; 2825 } 2826 2827 static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, 2828 unsigned int nr_pages) 2829 { 2830 if (mem_cgroup_is_root(memcg)) 2831 return 0; 2832 2833 return try_charge_memcg(memcg, gfp_mask, nr_pages); 2834 } 2835 2836 static inline void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages) 2837 { 2838 if (mem_cgroup_is_root(memcg)) 2839 return; 2840 2841 page_counter_uncharge(&memcg->memory, nr_pages); 2842 if (do_memsw_account()) 2843 page_counter_uncharge(&memcg->memsw, nr_pages); 2844 } 2845 2846 static void commit_charge(struct folio *folio, struct mem_cgroup *memcg) 2847 { 2848 VM_BUG_ON_FOLIO(folio_memcg(folio), folio); 2849 /* 2850 * Any of the following ensures page's memcg stability: 2851 * 2852 * - the page lock 2853 * - LRU isolation 2854 * - folio_memcg_lock() 2855 * - exclusive reference 2856 * - mem_cgroup_trylock_pages() 2857 */ 2858 folio->memcg_data = (unsigned long)memcg; 2859 } 2860 2861 #ifdef CONFIG_MEMCG_KMEM 2862 /* 2863 * The allocated objcg pointers array is not accounted directly. 2864 * Moreover, it should not come from DMA buffer and is not readily 2865 * reclaimable. So those GFP bits should be masked off. 2866 */ 2867 #define OBJCGS_CLEAR_MASK (__GFP_DMA | __GFP_RECLAIMABLE | __GFP_ACCOUNT) 2868 2869 /* 2870 * mod_objcg_mlstate() may be called with irq enabled, so 2871 * mod_memcg_lruvec_state() should be used. 2872 */ 2873 static inline void mod_objcg_mlstate(struct obj_cgroup *objcg, 2874 struct pglist_data *pgdat, 2875 enum node_stat_item idx, int nr) 2876 { 2877 struct mem_cgroup *memcg; 2878 struct lruvec *lruvec; 2879 2880 rcu_read_lock(); 2881 memcg = obj_cgroup_memcg(objcg); 2882 lruvec = mem_cgroup_lruvec(memcg, pgdat); 2883 mod_memcg_lruvec_state(lruvec, idx, nr); 2884 rcu_read_unlock(); 2885 } 2886 2887 int memcg_alloc_slab_cgroups(struct slab *slab, struct kmem_cache *s, 2888 gfp_t gfp, bool new_slab) 2889 { 2890 unsigned int objects = objs_per_slab(s, slab); 2891 unsigned long memcg_data; 2892 void *vec; 2893 2894 gfp &= ~OBJCGS_CLEAR_MASK; 2895 vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp, 2896 slab_nid(slab)); 2897 if (!vec) 2898 return -ENOMEM; 2899 2900 memcg_data = (unsigned long) vec | MEMCG_DATA_OBJCGS; 2901 if (new_slab) { 2902 /* 2903 * If the slab is brand new and nobody can yet access its 2904 * memcg_data, no synchronization is required and memcg_data can 2905 * be simply assigned. 2906 */ 2907 slab->memcg_data = memcg_data; 2908 } else if (cmpxchg(&slab->memcg_data, 0, memcg_data)) { 2909 /* 2910 * If the slab is already in use, somebody can allocate and 2911 * assign obj_cgroups in parallel. In this case the existing 2912 * objcg vector should be reused. 2913 */ 2914 kfree(vec); 2915 return 0; 2916 } 2917 2918 kmemleak_not_leak(vec); 2919 return 0; 2920 } 2921 2922 static __always_inline 2923 struct mem_cgroup *mem_cgroup_from_obj_folio(struct folio *folio, void *p) 2924 { 2925 /* 2926 * Slab objects are accounted individually, not per-page. 2927 * Memcg membership data for each individual object is saved in 2928 * slab->memcg_data. 2929 */ 2930 if (folio_test_slab(folio)) { 2931 struct obj_cgroup **objcgs; 2932 struct slab *slab; 2933 unsigned int off; 2934 2935 slab = folio_slab(folio); 2936 objcgs = slab_objcgs(slab); 2937 if (!objcgs) 2938 return NULL; 2939 2940 off = obj_to_index(slab->slab_cache, slab, p); 2941 if (objcgs[off]) 2942 return obj_cgroup_memcg(objcgs[off]); 2943 2944 return NULL; 2945 } 2946 2947 /* 2948 * folio_memcg_check() is used here, because in theory we can encounter 2949 * a folio where the slab flag has been cleared already, but 2950 * slab->memcg_data has not been freed yet 2951 * folio_memcg_check() will guarantee that a proper memory 2952 * cgroup pointer or NULL will be returned. 2953 */ 2954 return folio_memcg_check(folio); 2955 } 2956 2957 /* 2958 * Returns a pointer to the memory cgroup to which the kernel object is charged. 2959 * 2960 * A passed kernel object can be a slab object, vmalloc object or a generic 2961 * kernel page, so different mechanisms for getting the memory cgroup pointer 2962 * should be used. 2963 * 2964 * In certain cases (e.g. kernel stacks or large kmallocs with SLUB) the caller 2965 * can not know for sure how the kernel object is implemented. 2966 * mem_cgroup_from_obj() can be safely used in such cases. 2967 * 2968 * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(), 2969 * cgroup_mutex, etc. 2970 */ 2971 struct mem_cgroup *mem_cgroup_from_obj(void *p) 2972 { 2973 struct folio *folio; 2974 2975 if (mem_cgroup_disabled()) 2976 return NULL; 2977 2978 if (unlikely(is_vmalloc_addr(p))) 2979 folio = page_folio(vmalloc_to_page(p)); 2980 else 2981 folio = virt_to_folio(p); 2982 2983 return mem_cgroup_from_obj_folio(folio, p); 2984 } 2985 2986 /* 2987 * Returns a pointer to the memory cgroup to which the kernel object is charged. 2988 * Similar to mem_cgroup_from_obj(), but faster and not suitable for objects, 2989 * allocated using vmalloc(). 2990 * 2991 * A passed kernel object must be a slab object or a generic kernel page. 2992 * 2993 * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(), 2994 * cgroup_mutex, etc. 2995 */ 2996 struct mem_cgroup *mem_cgroup_from_slab_obj(void *p) 2997 { 2998 if (mem_cgroup_disabled()) 2999 return NULL; 3000 3001 return mem_cgroup_from_obj_folio(virt_to_folio(p), p); 3002 } 3003 3004 static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg) 3005 { 3006 struct obj_cgroup *objcg = NULL; 3007 3008 for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) { 3009 objcg = rcu_dereference(memcg->objcg); 3010 if (objcg && obj_cgroup_tryget(objcg)) 3011 break; 3012 objcg = NULL; 3013 } 3014 return objcg; 3015 } 3016 3017 __always_inline struct obj_cgroup *get_obj_cgroup_from_current(void) 3018 { 3019 struct obj_cgroup *objcg = NULL; 3020 struct mem_cgroup *memcg; 3021 3022 if (memcg_kmem_bypass()) 3023 return NULL; 3024 3025 rcu_read_lock(); 3026 if (unlikely(active_memcg())) 3027 memcg = active_memcg(); 3028 else 3029 memcg = mem_cgroup_from_task(current); 3030 objcg = __get_obj_cgroup_from_memcg(memcg); 3031 rcu_read_unlock(); 3032 return objcg; 3033 } 3034 3035 struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio) 3036 { 3037 struct obj_cgroup *objcg; 3038 3039 if (!memcg_kmem_online()) 3040 return NULL; 3041 3042 if (folio_memcg_kmem(folio)) { 3043 objcg = __folio_objcg(folio); 3044 obj_cgroup_get(objcg); 3045 } else { 3046 struct mem_cgroup *memcg; 3047 3048 rcu_read_lock(); 3049 memcg = __folio_memcg(folio); 3050 if (memcg) 3051 objcg = __get_obj_cgroup_from_memcg(memcg); 3052 else 3053 objcg = NULL; 3054 rcu_read_unlock(); 3055 } 3056 return objcg; 3057 } 3058 3059 static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages) 3060 { 3061 mod_memcg_state(memcg, MEMCG_KMEM, nr_pages); 3062 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) { 3063 if (nr_pages > 0) 3064 page_counter_charge(&memcg->kmem, nr_pages); 3065 else 3066 page_counter_uncharge(&memcg->kmem, -nr_pages); 3067 } 3068 } 3069 3070 3071 /* 3072 * obj_cgroup_uncharge_pages: uncharge a number of kernel pages from a objcg 3073 * @objcg: object cgroup to uncharge 3074 * @nr_pages: number of pages to uncharge 3075 */ 3076 static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg, 3077 unsigned int nr_pages) 3078 { 3079 struct mem_cgroup *memcg; 3080 3081 memcg = get_mem_cgroup_from_objcg(objcg); 3082 3083 memcg_account_kmem(memcg, -nr_pages); 3084 refill_stock(memcg, nr_pages); 3085 3086 css_put(&memcg->css); 3087 } 3088 3089 /* 3090 * obj_cgroup_charge_pages: charge a number of kernel pages to a objcg 3091 * @objcg: object cgroup to charge 3092 * @gfp: reclaim mode 3093 * @nr_pages: number of pages to charge 3094 * 3095 * Returns 0 on success, an error code on failure. 3096 */ 3097 static int obj_cgroup_charge_pages(struct obj_cgroup *objcg, gfp_t gfp, 3098 unsigned int nr_pages) 3099 { 3100 struct mem_cgroup *memcg; 3101 int ret; 3102 3103 memcg = get_mem_cgroup_from_objcg(objcg); 3104 3105 ret = try_charge_memcg(memcg, gfp, nr_pages); 3106 if (ret) 3107 goto out; 3108 3109 memcg_account_kmem(memcg, nr_pages); 3110 out: 3111 css_put(&memcg->css); 3112 3113 return ret; 3114 } 3115 3116 /** 3117 * __memcg_kmem_charge_page: charge a kmem page to the current memory cgroup 3118 * @page: page to charge 3119 * @gfp: reclaim mode 3120 * @order: allocation order 3121 * 3122 * Returns 0 on success, an error code on failure. 3123 */ 3124 int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order) 3125 { 3126 struct obj_cgroup *objcg; 3127 int ret = 0; 3128 3129 objcg = get_obj_cgroup_from_current(); 3130 if (objcg) { 3131 ret = obj_cgroup_charge_pages(objcg, gfp, 1 << order); 3132 if (!ret) { 3133 page->memcg_data = (unsigned long)objcg | 3134 MEMCG_DATA_KMEM; 3135 return 0; 3136 } 3137 obj_cgroup_put(objcg); 3138 } 3139 return ret; 3140 } 3141 3142 /** 3143 * __memcg_kmem_uncharge_page: uncharge a kmem page 3144 * @page: page to uncharge 3145 * @order: allocation order 3146 */ 3147 void __memcg_kmem_uncharge_page(struct page *page, int order) 3148 { 3149 struct folio *folio = page_folio(page); 3150 struct obj_cgroup *objcg; 3151 unsigned int nr_pages = 1 << order; 3152 3153 if (!folio_memcg_kmem(folio)) 3154 return; 3155 3156 objcg = __folio_objcg(folio); 3157 obj_cgroup_uncharge_pages(objcg, nr_pages); 3158 folio->memcg_data = 0; 3159 obj_cgroup_put(objcg); 3160 } 3161 3162 void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat, 3163 enum node_stat_item idx, int nr) 3164 { 3165 struct memcg_stock_pcp *stock; 3166 struct obj_cgroup *old = NULL; 3167 unsigned long flags; 3168 int *bytes; 3169 3170 local_lock_irqsave(&memcg_stock.stock_lock, flags); 3171 stock = this_cpu_ptr(&memcg_stock); 3172 3173 /* 3174 * Save vmstat data in stock and skip vmstat array update unless 3175 * accumulating over a page of vmstat data or when pgdat or idx 3176 * changes. 3177 */ 3178 if (READ_ONCE(stock->cached_objcg) != objcg) { 3179 old = drain_obj_stock(stock); 3180 obj_cgroup_get(objcg); 3181 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes) 3182 ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0; 3183 WRITE_ONCE(stock->cached_objcg, objcg); 3184 stock->cached_pgdat = pgdat; 3185 } else if (stock->cached_pgdat != pgdat) { 3186 /* Flush the existing cached vmstat data */ 3187 struct pglist_data *oldpg = stock->cached_pgdat; 3188 3189 if (stock->nr_slab_reclaimable_b) { 3190 mod_objcg_mlstate(objcg, oldpg, NR_SLAB_RECLAIMABLE_B, 3191 stock->nr_slab_reclaimable_b); 3192 stock->nr_slab_reclaimable_b = 0; 3193 } 3194 if (stock->nr_slab_unreclaimable_b) { 3195 mod_objcg_mlstate(objcg, oldpg, NR_SLAB_UNRECLAIMABLE_B, 3196 stock->nr_slab_unreclaimable_b); 3197 stock->nr_slab_unreclaimable_b = 0; 3198 } 3199 stock->cached_pgdat = pgdat; 3200 } 3201 3202 bytes = (idx == NR_SLAB_RECLAIMABLE_B) ? &stock->nr_slab_reclaimable_b 3203 : &stock->nr_slab_unreclaimable_b; 3204 /* 3205 * Even for large object >= PAGE_SIZE, the vmstat data will still be 3206 * cached locally at least once before pushing it out. 3207 */ 3208 if (!*bytes) { 3209 *bytes = nr; 3210 nr = 0; 3211 } else { 3212 *bytes += nr; 3213 if (abs(*bytes) > PAGE_SIZE) { 3214 nr = *bytes; 3215 *bytes = 0; 3216 } else { 3217 nr = 0; 3218 } 3219 } 3220 if (nr) 3221 mod_objcg_mlstate(objcg, pgdat, idx, nr); 3222 3223 local_unlock_irqrestore(&memcg_stock.stock_lock, flags); 3224 if (old) 3225 obj_cgroup_put(old); 3226 } 3227 3228 static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes) 3229 { 3230 struct memcg_stock_pcp *stock; 3231 unsigned long flags; 3232 bool ret = false; 3233 3234 local_lock_irqsave(&memcg_stock.stock_lock, flags); 3235 3236 stock = this_cpu_ptr(&memcg_stock); 3237 if (objcg == READ_ONCE(stock->cached_objcg) && stock->nr_bytes >= nr_bytes) { 3238 stock->nr_bytes -= nr_bytes; 3239 ret = true; 3240 } 3241 3242 local_unlock_irqrestore(&memcg_stock.stock_lock, flags); 3243 3244 return ret; 3245 } 3246 3247 static struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock) 3248 { 3249 struct obj_cgroup *old = READ_ONCE(stock->cached_objcg); 3250 3251 if (!old) 3252 return NULL; 3253 3254 if (stock->nr_bytes) { 3255 unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT; 3256 unsigned int nr_bytes = stock->nr_bytes & (PAGE_SIZE - 1); 3257 3258 if (nr_pages) { 3259 struct mem_cgroup *memcg; 3260 3261 memcg = get_mem_cgroup_from_objcg(old); 3262 3263 memcg_account_kmem(memcg, -nr_pages); 3264 __refill_stock(memcg, nr_pages); 3265 3266 css_put(&memcg->css); 3267 } 3268 3269 /* 3270 * The leftover is flushed to the centralized per-memcg value. 3271 * On the next attempt to refill obj stock it will be moved 3272 * to a per-cpu stock (probably, on an other CPU), see 3273 * refill_obj_stock(). 3274 * 3275 * How often it's flushed is a trade-off between the memory 3276 * limit enforcement accuracy and potential CPU contention, 3277 * so it might be changed in the future. 3278 */ 3279 atomic_add(nr_bytes, &old->nr_charged_bytes); 3280 stock->nr_bytes = 0; 3281 } 3282 3283 /* 3284 * Flush the vmstat data in current stock 3285 */ 3286 if (stock->nr_slab_reclaimable_b || stock->nr_slab_unreclaimable_b) { 3287 if (stock->nr_slab_reclaimable_b) { 3288 mod_objcg_mlstate(old, stock->cached_pgdat, 3289 NR_SLAB_RECLAIMABLE_B, 3290 stock->nr_slab_reclaimable_b); 3291 stock->nr_slab_reclaimable_b = 0; 3292 } 3293 if (stock->nr_slab_unreclaimable_b) { 3294 mod_objcg_mlstate(old, stock->cached_pgdat, 3295 NR_SLAB_UNRECLAIMABLE_B, 3296 stock->nr_slab_unreclaimable_b); 3297 stock->nr_slab_unreclaimable_b = 0; 3298 } 3299 stock->cached_pgdat = NULL; 3300 } 3301 3302 WRITE_ONCE(stock->cached_objcg, NULL); 3303 /* 3304 * The `old' objects needs to be released by the caller via 3305 * obj_cgroup_put() outside of memcg_stock_pcp::stock_lock. 3306 */ 3307 return old; 3308 } 3309 3310 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock, 3311 struct mem_cgroup *root_memcg) 3312 { 3313 struct obj_cgroup *objcg = READ_ONCE(stock->cached_objcg); 3314 struct mem_cgroup *memcg; 3315 3316 if (objcg) { 3317 memcg = obj_cgroup_memcg(objcg); 3318 if (memcg && mem_cgroup_is_descendant(memcg, root_memcg)) 3319 return true; 3320 } 3321 3322 return false; 3323 } 3324 3325 static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes, 3326 bool allow_uncharge) 3327 { 3328 struct memcg_stock_pcp *stock; 3329 struct obj_cgroup *old = NULL; 3330 unsigned long flags; 3331 unsigned int nr_pages = 0; 3332 3333 local_lock_irqsave(&memcg_stock.stock_lock, flags); 3334 3335 stock = this_cpu_ptr(&memcg_stock); 3336 if (READ_ONCE(stock->cached_objcg) != objcg) { /* reset if necessary */ 3337 old = drain_obj_stock(stock); 3338 obj_cgroup_get(objcg); 3339 WRITE_ONCE(stock->cached_objcg, objcg); 3340 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes) 3341 ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0; 3342 allow_uncharge = true; /* Allow uncharge when objcg changes */ 3343 } 3344 stock->nr_bytes += nr_bytes; 3345 3346 if (allow_uncharge && (stock->nr_bytes > PAGE_SIZE)) { 3347 nr_pages = stock->nr_bytes >> PAGE_SHIFT; 3348 stock->nr_bytes &= (PAGE_SIZE - 1); 3349 } 3350 3351 local_unlock_irqrestore(&memcg_stock.stock_lock, flags); 3352 if (old) 3353 obj_cgroup_put(old); 3354 3355 if (nr_pages) 3356 obj_cgroup_uncharge_pages(objcg, nr_pages); 3357 } 3358 3359 int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size) 3360 { 3361 unsigned int nr_pages, nr_bytes; 3362 int ret; 3363 3364 if (consume_obj_stock(objcg, size)) 3365 return 0; 3366 3367 /* 3368 * In theory, objcg->nr_charged_bytes can have enough 3369 * pre-charged bytes to satisfy the allocation. However, 3370 * flushing objcg->nr_charged_bytes requires two atomic 3371 * operations, and objcg->nr_charged_bytes can't be big. 3372 * The shared objcg->nr_charged_bytes can also become a 3373 * performance bottleneck if all tasks of the same memcg are 3374 * trying to update it. So it's better to ignore it and try 3375 * grab some new pages. The stock's nr_bytes will be flushed to 3376 * objcg->nr_charged_bytes later on when objcg changes. 3377 * 3378 * The stock's nr_bytes may contain enough pre-charged bytes 3379 * to allow one less page from being charged, but we can't rely 3380 * on the pre-charged bytes not being changed outside of 3381 * consume_obj_stock() or refill_obj_stock(). So ignore those 3382 * pre-charged bytes as well when charging pages. To avoid a 3383 * page uncharge right after a page charge, we set the 3384 * allow_uncharge flag to false when calling refill_obj_stock() 3385 * to temporarily allow the pre-charged bytes to exceed the page 3386 * size limit. The maximum reachable value of the pre-charged 3387 * bytes is (sizeof(object) + PAGE_SIZE - 2) if there is no data 3388 * race. 3389 */ 3390 nr_pages = size >> PAGE_SHIFT; 3391 nr_bytes = size & (PAGE_SIZE - 1); 3392 3393 if (nr_bytes) 3394 nr_pages += 1; 3395 3396 ret = obj_cgroup_charge_pages(objcg, gfp, nr_pages); 3397 if (!ret && nr_bytes) 3398 refill_obj_stock(objcg, PAGE_SIZE - nr_bytes, false); 3399 3400 return ret; 3401 } 3402 3403 void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size) 3404 { 3405 refill_obj_stock(objcg, size, true); 3406 } 3407 3408 #endif /* CONFIG_MEMCG_KMEM */ 3409 3410 /* 3411 * Because page_memcg(head) is not set on tails, set it now. 3412 */ 3413 void split_page_memcg(struct page *head, unsigned int nr) 3414 { 3415 struct folio *folio = page_folio(head); 3416 struct mem_cgroup *memcg = folio_memcg(folio); 3417 int i; 3418 3419 if (mem_cgroup_disabled() || !memcg) 3420 return; 3421 3422 for (i = 1; i < nr; i++) 3423 folio_page(folio, i)->memcg_data = folio->memcg_data; 3424 3425 if (folio_memcg_kmem(folio)) 3426 obj_cgroup_get_many(__folio_objcg(folio), nr - 1); 3427 else 3428 css_get_many(&memcg->css, nr - 1); 3429 } 3430 3431 #ifdef CONFIG_SWAP 3432 /** 3433 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record. 3434 * @entry: swap entry to be moved 3435 * @from: mem_cgroup which the entry is moved from 3436 * @to: mem_cgroup which the entry is moved to 3437 * 3438 * It succeeds only when the swap_cgroup's record for this entry is the same 3439 * as the mem_cgroup's id of @from. 3440 * 3441 * Returns 0 on success, -EINVAL on failure. 3442 * 3443 * The caller must have charged to @to, IOW, called page_counter_charge() about 3444 * both res and memsw, and called css_get(). 3445 */ 3446 static int mem_cgroup_move_swap_account(swp_entry_t entry, 3447 struct mem_cgroup *from, struct mem_cgroup *to) 3448 { 3449 unsigned short old_id, new_id; 3450 3451 old_id = mem_cgroup_id(from); 3452 new_id = mem_cgroup_id(to); 3453 3454 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) { 3455 mod_memcg_state(from, MEMCG_SWAP, -1); 3456 mod_memcg_state(to, MEMCG_SWAP, 1); 3457 return 0; 3458 } 3459 return -EINVAL; 3460 } 3461 #else 3462 static inline int mem_cgroup_move_swap_account(swp_entry_t entry, 3463 struct mem_cgroup *from, struct mem_cgroup *to) 3464 { 3465 return -EINVAL; 3466 } 3467 #endif 3468 3469 static DEFINE_MUTEX(memcg_max_mutex); 3470 3471 static int mem_cgroup_resize_max(struct mem_cgroup *memcg, 3472 unsigned long max, bool memsw) 3473 { 3474 bool enlarge = false; 3475 bool drained = false; 3476 int ret; 3477 bool limits_invariant; 3478 struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory; 3479 3480 do { 3481 if (signal_pending(current)) { 3482 ret = -EINTR; 3483 break; 3484 } 3485 3486 mutex_lock(&memcg_max_mutex); 3487 /* 3488 * Make sure that the new limit (memsw or memory limit) doesn't 3489 * break our basic invariant rule memory.max <= memsw.max. 3490 */ 3491 limits_invariant = memsw ? max >= READ_ONCE(memcg->memory.max) : 3492 max <= memcg->memsw.max; 3493 if (!limits_invariant) { 3494 mutex_unlock(&memcg_max_mutex); 3495 ret = -EINVAL; 3496 break; 3497 } 3498 if (max > counter->max) 3499 enlarge = true; 3500 ret = page_counter_set_max(counter, max); 3501 mutex_unlock(&memcg_max_mutex); 3502 3503 if (!ret) 3504 break; 3505 3506 if (!drained) { 3507 drain_all_stock(memcg); 3508 drained = true; 3509 continue; 3510 } 3511 3512 if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, 3513 memsw ? 0 : MEMCG_RECLAIM_MAY_SWAP)) { 3514 ret = -EBUSY; 3515 break; 3516 } 3517 } while (true); 3518 3519 if (!ret && enlarge) 3520 memcg_oom_recover(memcg); 3521 3522 return ret; 3523 } 3524 3525 unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order, 3526 gfp_t gfp_mask, 3527 unsigned long *total_scanned) 3528 { 3529 unsigned long nr_reclaimed = 0; 3530 struct mem_cgroup_per_node *mz, *next_mz = NULL; 3531 unsigned long reclaimed; 3532 int loop = 0; 3533 struct mem_cgroup_tree_per_node *mctz; 3534 unsigned long excess; 3535 3536 if (lru_gen_enabled()) 3537 return 0; 3538 3539 if (order > 0) 3540 return 0; 3541 3542 mctz = soft_limit_tree.rb_tree_per_node[pgdat->node_id]; 3543 3544 /* 3545 * Do not even bother to check the largest node if the root 3546 * is empty. Do it lockless to prevent lock bouncing. Races 3547 * are acceptable as soft limit is best effort anyway. 3548 */ 3549 if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root)) 3550 return 0; 3551 3552 /* 3553 * This loop can run a while, specially if mem_cgroup's continuously 3554 * keep exceeding their soft limit and putting the system under 3555 * pressure 3556 */ 3557 do { 3558 if (next_mz) 3559 mz = next_mz; 3560 else 3561 mz = mem_cgroup_largest_soft_limit_node(mctz); 3562 if (!mz) 3563 break; 3564 3565 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat, 3566 gfp_mask, total_scanned); 3567 nr_reclaimed += reclaimed; 3568 spin_lock_irq(&mctz->lock); 3569 3570 /* 3571 * If we failed to reclaim anything from this memory cgroup 3572 * it is time to move on to the next cgroup 3573 */ 3574 next_mz = NULL; 3575 if (!reclaimed) 3576 next_mz = __mem_cgroup_largest_soft_limit_node(mctz); 3577 3578 excess = soft_limit_excess(mz->memcg); 3579 /* 3580 * One school of thought says that we should not add 3581 * back the node to the tree if reclaim returns 0. 3582 * But our reclaim could return 0, simply because due 3583 * to priority we are exposing a smaller subset of 3584 * memory to reclaim from. Consider this as a longer 3585 * term TODO. 3586 */ 3587 /* If excess == 0, no tree ops */ 3588 __mem_cgroup_insert_exceeded(mz, mctz, excess); 3589 spin_unlock_irq(&mctz->lock); 3590 css_put(&mz->memcg->css); 3591 loop++; 3592 /* 3593 * Could not reclaim anything and there are no more 3594 * mem cgroups to try or we seem to be looping without 3595 * reclaiming anything. 3596 */ 3597 if (!nr_reclaimed && 3598 (next_mz == NULL || 3599 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS)) 3600 break; 3601 } while (!nr_reclaimed); 3602 if (next_mz) 3603 css_put(&next_mz->memcg->css); 3604 return nr_reclaimed; 3605 } 3606 3607 /* 3608 * Reclaims as many pages from the given memcg as possible. 3609 * 3610 * Caller is responsible for holding css reference for memcg. 3611 */ 3612 static int mem_cgroup_force_empty(struct mem_cgroup *memcg) 3613 { 3614 int nr_retries = MAX_RECLAIM_RETRIES; 3615 3616 /* we call try-to-free pages for make this cgroup empty */ 3617 lru_add_drain_all(); 3618 3619 drain_all_stock(memcg); 3620 3621 /* try to free all pages in this cgroup */ 3622 while (nr_retries && page_counter_read(&memcg->memory)) { 3623 if (signal_pending(current)) 3624 return -EINTR; 3625 3626 if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, 3627 MEMCG_RECLAIM_MAY_SWAP)) 3628 nr_retries--; 3629 } 3630 3631 return 0; 3632 } 3633 3634 static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of, 3635 char *buf, size_t nbytes, 3636 loff_t off) 3637 { 3638 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 3639 3640 if (mem_cgroup_is_root(memcg)) 3641 return -EINVAL; 3642 return mem_cgroup_force_empty(memcg) ?: nbytes; 3643 } 3644 3645 static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css, 3646 struct cftype *cft) 3647 { 3648 return 1; 3649 } 3650 3651 static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css, 3652 struct cftype *cft, u64 val) 3653 { 3654 if (val == 1) 3655 return 0; 3656 3657 pr_warn_once("Non-hierarchical mode is deprecated. " 3658 "Please report your usecase to linux-mm@kvack.org if you " 3659 "depend on this functionality.\n"); 3660 3661 return -EINVAL; 3662 } 3663 3664 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap) 3665 { 3666 unsigned long val; 3667 3668 if (mem_cgroup_is_root(memcg)) { 3669 /* 3670 * Approximate root's usage from global state. This isn't 3671 * perfect, but the root usage was always an approximation. 3672 */ 3673 val = global_node_page_state(NR_FILE_PAGES) + 3674 global_node_page_state(NR_ANON_MAPPED); 3675 if (swap) 3676 val += total_swap_pages - get_nr_swap_pages(); 3677 } else { 3678 if (!swap) 3679 val = page_counter_read(&memcg->memory); 3680 else 3681 val = page_counter_read(&memcg->memsw); 3682 } 3683 return val; 3684 } 3685 3686 enum { 3687 RES_USAGE, 3688 RES_LIMIT, 3689 RES_MAX_USAGE, 3690 RES_FAILCNT, 3691 RES_SOFT_LIMIT, 3692 }; 3693 3694 static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css, 3695 struct cftype *cft) 3696 { 3697 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 3698 struct page_counter *counter; 3699 3700 switch (MEMFILE_TYPE(cft->private)) { 3701 case _MEM: 3702 counter = &memcg->memory; 3703 break; 3704 case _MEMSWAP: 3705 counter = &memcg->memsw; 3706 break; 3707 case _KMEM: 3708 counter = &memcg->kmem; 3709 break; 3710 case _TCP: 3711 counter = &memcg->tcpmem; 3712 break; 3713 default: 3714 BUG(); 3715 } 3716 3717 switch (MEMFILE_ATTR(cft->private)) { 3718 case RES_USAGE: 3719 if (counter == &memcg->memory) 3720 return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE; 3721 if (counter == &memcg->memsw) 3722 return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE; 3723 return (u64)page_counter_read(counter) * PAGE_SIZE; 3724 case RES_LIMIT: 3725 return (u64)counter->max * PAGE_SIZE; 3726 case RES_MAX_USAGE: 3727 return (u64)counter->watermark * PAGE_SIZE; 3728 case RES_FAILCNT: 3729 return counter->failcnt; 3730 case RES_SOFT_LIMIT: 3731 return (u64)READ_ONCE(memcg->soft_limit) * PAGE_SIZE; 3732 default: 3733 BUG(); 3734 } 3735 } 3736 3737 /* 3738 * This function doesn't do anything useful. Its only job is to provide a read 3739 * handler for a file so that cgroup_file_mode() will add read permissions. 3740 */ 3741 static int mem_cgroup_dummy_seq_show(__always_unused struct seq_file *m, 3742 __always_unused void *v) 3743 { 3744 return -EINVAL; 3745 } 3746 3747 #ifdef CONFIG_MEMCG_KMEM 3748 static int memcg_online_kmem(struct mem_cgroup *memcg) 3749 { 3750 struct obj_cgroup *objcg; 3751 3752 if (mem_cgroup_kmem_disabled()) 3753 return 0; 3754 3755 if (unlikely(mem_cgroup_is_root(memcg))) 3756 return 0; 3757 3758 objcg = obj_cgroup_alloc(); 3759 if (!objcg) 3760 return -ENOMEM; 3761 3762 objcg->memcg = memcg; 3763 rcu_assign_pointer(memcg->objcg, objcg); 3764 3765 static_branch_enable(&memcg_kmem_online_key); 3766 3767 memcg->kmemcg_id = memcg->id.id; 3768 3769 return 0; 3770 } 3771 3772 static void memcg_offline_kmem(struct mem_cgroup *memcg) 3773 { 3774 struct mem_cgroup *parent; 3775 3776 if (mem_cgroup_kmem_disabled()) 3777 return; 3778 3779 if (unlikely(mem_cgroup_is_root(memcg))) 3780 return; 3781 3782 parent = parent_mem_cgroup(memcg); 3783 if (!parent) 3784 parent = root_mem_cgroup; 3785 3786 memcg_reparent_objcgs(memcg, parent); 3787 3788 /* 3789 * After we have finished memcg_reparent_objcgs(), all list_lrus 3790 * corresponding to this cgroup are guaranteed to remain empty. 3791 * The ordering is imposed by list_lru_node->lock taken by 3792 * memcg_reparent_list_lrus(). 3793 */ 3794 memcg_reparent_list_lrus(memcg, parent); 3795 } 3796 #else 3797 static int memcg_online_kmem(struct mem_cgroup *memcg) 3798 { 3799 return 0; 3800 } 3801 static void memcg_offline_kmem(struct mem_cgroup *memcg) 3802 { 3803 } 3804 #endif /* CONFIG_MEMCG_KMEM */ 3805 3806 static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max) 3807 { 3808 int ret; 3809 3810 mutex_lock(&memcg_max_mutex); 3811 3812 ret = page_counter_set_max(&memcg->tcpmem, max); 3813 if (ret) 3814 goto out; 3815 3816 if (!memcg->tcpmem_active) { 3817 /* 3818 * The active flag needs to be written after the static_key 3819 * update. This is what guarantees that the socket activation 3820 * function is the last one to run. See mem_cgroup_sk_alloc() 3821 * for details, and note that we don't mark any socket as 3822 * belonging to this memcg until that flag is up. 3823 * 3824 * We need to do this, because static_keys will span multiple 3825 * sites, but we can't control their order. If we mark a socket 3826 * as accounted, but the accounting functions are not patched in 3827 * yet, we'll lose accounting. 3828 * 3829 * We never race with the readers in mem_cgroup_sk_alloc(), 3830 * because when this value change, the code to process it is not 3831 * patched in yet. 3832 */ 3833 static_branch_inc(&memcg_sockets_enabled_key); 3834 memcg->tcpmem_active = true; 3835 } 3836 out: 3837 mutex_unlock(&memcg_max_mutex); 3838 return ret; 3839 } 3840 3841 /* 3842 * The user of this function is... 3843 * RES_LIMIT. 3844 */ 3845 static ssize_t mem_cgroup_write(struct kernfs_open_file *of, 3846 char *buf, size_t nbytes, loff_t off) 3847 { 3848 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 3849 unsigned long nr_pages; 3850 int ret; 3851 3852 buf = strstrip(buf); 3853 ret = page_counter_memparse(buf, "-1", &nr_pages); 3854 if (ret) 3855 return ret; 3856 3857 switch (MEMFILE_ATTR(of_cft(of)->private)) { 3858 case RES_LIMIT: 3859 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */ 3860 ret = -EINVAL; 3861 break; 3862 } 3863 switch (MEMFILE_TYPE(of_cft(of)->private)) { 3864 case _MEM: 3865 ret = mem_cgroup_resize_max(memcg, nr_pages, false); 3866 break; 3867 case _MEMSWAP: 3868 ret = mem_cgroup_resize_max(memcg, nr_pages, true); 3869 break; 3870 case _KMEM: 3871 pr_warn_once("kmem.limit_in_bytes is deprecated and will be removed. " 3872 "Writing any value to this file has no effect. " 3873 "Please report your usecase to linux-mm@kvack.org if you " 3874 "depend on this functionality.\n"); 3875 ret = 0; 3876 break; 3877 case _TCP: 3878 ret = memcg_update_tcp_max(memcg, nr_pages); 3879 break; 3880 } 3881 break; 3882 case RES_SOFT_LIMIT: 3883 if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 3884 ret = -EOPNOTSUPP; 3885 } else { 3886 WRITE_ONCE(memcg->soft_limit, nr_pages); 3887 ret = 0; 3888 } 3889 break; 3890 } 3891 return ret ?: nbytes; 3892 } 3893 3894 static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf, 3895 size_t nbytes, loff_t off) 3896 { 3897 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 3898 struct page_counter *counter; 3899 3900 switch (MEMFILE_TYPE(of_cft(of)->private)) { 3901 case _MEM: 3902 counter = &memcg->memory; 3903 break; 3904 case _MEMSWAP: 3905 counter = &memcg->memsw; 3906 break; 3907 case _KMEM: 3908 counter = &memcg->kmem; 3909 break; 3910 case _TCP: 3911 counter = &memcg->tcpmem; 3912 break; 3913 default: 3914 BUG(); 3915 } 3916 3917 switch (MEMFILE_ATTR(of_cft(of)->private)) { 3918 case RES_MAX_USAGE: 3919 page_counter_reset_watermark(counter); 3920 break; 3921 case RES_FAILCNT: 3922 counter->failcnt = 0; 3923 break; 3924 default: 3925 BUG(); 3926 } 3927 3928 return nbytes; 3929 } 3930 3931 static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css, 3932 struct cftype *cft) 3933 { 3934 return mem_cgroup_from_css(css)->move_charge_at_immigrate; 3935 } 3936 3937 #ifdef CONFIG_MMU 3938 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css, 3939 struct cftype *cft, u64 val) 3940 { 3941 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 3942 3943 pr_warn_once("Cgroup memory moving (move_charge_at_immigrate) is deprecated. " 3944 "Please report your usecase to linux-mm@kvack.org if you " 3945 "depend on this functionality.\n"); 3946 3947 if (val & ~MOVE_MASK) 3948 return -EINVAL; 3949 3950 /* 3951 * No kind of locking is needed in here, because ->can_attach() will 3952 * check this value once in the beginning of the process, and then carry 3953 * on with stale data. This means that changes to this value will only 3954 * affect task migrations starting after the change. 3955 */ 3956 memcg->move_charge_at_immigrate = val; 3957 return 0; 3958 } 3959 #else 3960 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css, 3961 struct cftype *cft, u64 val) 3962 { 3963 return -ENOSYS; 3964 } 3965 #endif 3966 3967 #ifdef CONFIG_NUMA 3968 3969 #define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE)) 3970 #define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON)) 3971 #define LRU_ALL ((1 << NR_LRU_LISTS) - 1) 3972 3973 static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg, 3974 int nid, unsigned int lru_mask, bool tree) 3975 { 3976 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid)); 3977 unsigned long nr = 0; 3978 enum lru_list lru; 3979 3980 VM_BUG_ON((unsigned)nid >= nr_node_ids); 3981 3982 for_each_lru(lru) { 3983 if (!(BIT(lru) & lru_mask)) 3984 continue; 3985 if (tree) 3986 nr += lruvec_page_state(lruvec, NR_LRU_BASE + lru); 3987 else 3988 nr += lruvec_page_state_local(lruvec, NR_LRU_BASE + lru); 3989 } 3990 return nr; 3991 } 3992 3993 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg, 3994 unsigned int lru_mask, 3995 bool tree) 3996 { 3997 unsigned long nr = 0; 3998 enum lru_list lru; 3999 4000 for_each_lru(lru) { 4001 if (!(BIT(lru) & lru_mask)) 4002 continue; 4003 if (tree) 4004 nr += memcg_page_state(memcg, NR_LRU_BASE + lru); 4005 else 4006 nr += memcg_page_state_local(memcg, NR_LRU_BASE + lru); 4007 } 4008 return nr; 4009 } 4010 4011 static int memcg_numa_stat_show(struct seq_file *m, void *v) 4012 { 4013 struct numa_stat { 4014 const char *name; 4015 unsigned int lru_mask; 4016 }; 4017 4018 static const struct numa_stat stats[] = { 4019 { "total", LRU_ALL }, 4020 { "file", LRU_ALL_FILE }, 4021 { "anon", LRU_ALL_ANON }, 4022 { "unevictable", BIT(LRU_UNEVICTABLE) }, 4023 }; 4024 const struct numa_stat *stat; 4025 int nid; 4026 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 4027 4028 mem_cgroup_flush_stats(); 4029 4030 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) { 4031 seq_printf(m, "%s=%lu", stat->name, 4032 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask, 4033 false)); 4034 for_each_node_state(nid, N_MEMORY) 4035 seq_printf(m, " N%d=%lu", nid, 4036 mem_cgroup_node_nr_lru_pages(memcg, nid, 4037 stat->lru_mask, false)); 4038 seq_putc(m, '\n'); 4039 } 4040 4041 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) { 4042 4043 seq_printf(m, "hierarchical_%s=%lu", stat->name, 4044 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask, 4045 true)); 4046 for_each_node_state(nid, N_MEMORY) 4047 seq_printf(m, " N%d=%lu", nid, 4048 mem_cgroup_node_nr_lru_pages(memcg, nid, 4049 stat->lru_mask, true)); 4050 seq_putc(m, '\n'); 4051 } 4052 4053 return 0; 4054 } 4055 #endif /* CONFIG_NUMA */ 4056 4057 static const unsigned int memcg1_stats[] = { 4058 NR_FILE_PAGES, 4059 NR_ANON_MAPPED, 4060 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4061 NR_ANON_THPS, 4062 #endif 4063 NR_SHMEM, 4064 NR_FILE_MAPPED, 4065 NR_FILE_DIRTY, 4066 NR_WRITEBACK, 4067 WORKINGSET_REFAULT_ANON, 4068 WORKINGSET_REFAULT_FILE, 4069 MEMCG_SWAP, 4070 }; 4071 4072 static const char *const memcg1_stat_names[] = { 4073 "cache", 4074 "rss", 4075 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4076 "rss_huge", 4077 #endif 4078 "shmem", 4079 "mapped_file", 4080 "dirty", 4081 "writeback", 4082 "workingset_refault_anon", 4083 "workingset_refault_file", 4084 "swap", 4085 }; 4086 4087 /* Universal VM events cgroup1 shows, original sort order */ 4088 static const unsigned int memcg1_events[] = { 4089 PGPGIN, 4090 PGPGOUT, 4091 PGFAULT, 4092 PGMAJFAULT, 4093 }; 4094 4095 static void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s) 4096 { 4097 unsigned long memory, memsw; 4098 struct mem_cgroup *mi; 4099 unsigned int i; 4100 4101 BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats)); 4102 4103 mem_cgroup_flush_stats(); 4104 4105 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) { 4106 unsigned long nr; 4107 4108 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account()) 4109 continue; 4110 nr = memcg_page_state_local(memcg, memcg1_stats[i]); 4111 seq_buf_printf(s, "%s %lu\n", memcg1_stat_names[i], 4112 nr * memcg_page_state_unit(memcg1_stats[i])); 4113 } 4114 4115 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++) 4116 seq_buf_printf(s, "%s %lu\n", vm_event_name(memcg1_events[i]), 4117 memcg_events_local(memcg, memcg1_events[i])); 4118 4119 for (i = 0; i < NR_LRU_LISTS; i++) 4120 seq_buf_printf(s, "%s %lu\n", lru_list_name(i), 4121 memcg_page_state_local(memcg, NR_LRU_BASE + i) * 4122 PAGE_SIZE); 4123 4124 /* Hierarchical information */ 4125 memory = memsw = PAGE_COUNTER_MAX; 4126 for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) { 4127 memory = min(memory, READ_ONCE(mi->memory.max)); 4128 memsw = min(memsw, READ_ONCE(mi->memsw.max)); 4129 } 4130 seq_buf_printf(s, "hierarchical_memory_limit %llu\n", 4131 (u64)memory * PAGE_SIZE); 4132 if (do_memsw_account()) 4133 seq_buf_printf(s, "hierarchical_memsw_limit %llu\n", 4134 (u64)memsw * PAGE_SIZE); 4135 4136 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) { 4137 unsigned long nr; 4138 4139 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account()) 4140 continue; 4141 nr = memcg_page_state(memcg, memcg1_stats[i]); 4142 seq_buf_printf(s, "total_%s %llu\n", memcg1_stat_names[i], 4143 (u64)nr * memcg_page_state_unit(memcg1_stats[i])); 4144 } 4145 4146 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++) 4147 seq_buf_printf(s, "total_%s %llu\n", 4148 vm_event_name(memcg1_events[i]), 4149 (u64)memcg_events(memcg, memcg1_events[i])); 4150 4151 for (i = 0; i < NR_LRU_LISTS; i++) 4152 seq_buf_printf(s, "total_%s %llu\n", lru_list_name(i), 4153 (u64)memcg_page_state(memcg, NR_LRU_BASE + i) * 4154 PAGE_SIZE); 4155 4156 #ifdef CONFIG_DEBUG_VM 4157 { 4158 pg_data_t *pgdat; 4159 struct mem_cgroup_per_node *mz; 4160 unsigned long anon_cost = 0; 4161 unsigned long file_cost = 0; 4162 4163 for_each_online_pgdat(pgdat) { 4164 mz = memcg->nodeinfo[pgdat->node_id]; 4165 4166 anon_cost += mz->lruvec.anon_cost; 4167 file_cost += mz->lruvec.file_cost; 4168 } 4169 seq_buf_printf(s, "anon_cost %lu\n", anon_cost); 4170 seq_buf_printf(s, "file_cost %lu\n", file_cost); 4171 } 4172 #endif 4173 } 4174 4175 static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css, 4176 struct cftype *cft) 4177 { 4178 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4179 4180 return mem_cgroup_swappiness(memcg); 4181 } 4182 4183 static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css, 4184 struct cftype *cft, u64 val) 4185 { 4186 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4187 4188 if (val > 200) 4189 return -EINVAL; 4190 4191 if (!mem_cgroup_is_root(memcg)) 4192 WRITE_ONCE(memcg->swappiness, val); 4193 else 4194 WRITE_ONCE(vm_swappiness, val); 4195 4196 return 0; 4197 } 4198 4199 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap) 4200 { 4201 struct mem_cgroup_threshold_ary *t; 4202 unsigned long usage; 4203 int i; 4204 4205 rcu_read_lock(); 4206 if (!swap) 4207 t = rcu_dereference(memcg->thresholds.primary); 4208 else 4209 t = rcu_dereference(memcg->memsw_thresholds.primary); 4210 4211 if (!t) 4212 goto unlock; 4213 4214 usage = mem_cgroup_usage(memcg, swap); 4215 4216 /* 4217 * current_threshold points to threshold just below or equal to usage. 4218 * If it's not true, a threshold was crossed after last 4219 * call of __mem_cgroup_threshold(). 4220 */ 4221 i = t->current_threshold; 4222 4223 /* 4224 * Iterate backward over array of thresholds starting from 4225 * current_threshold and check if a threshold is crossed. 4226 * If none of thresholds below usage is crossed, we read 4227 * only one element of the array here. 4228 */ 4229 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--) 4230 eventfd_signal(t->entries[i].eventfd, 1); 4231 4232 /* i = current_threshold + 1 */ 4233 i++; 4234 4235 /* 4236 * Iterate forward over array of thresholds starting from 4237 * current_threshold+1 and check if a threshold is crossed. 4238 * If none of thresholds above usage is crossed, we read 4239 * only one element of the array here. 4240 */ 4241 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++) 4242 eventfd_signal(t->entries[i].eventfd, 1); 4243 4244 /* Update current_threshold */ 4245 t->current_threshold = i - 1; 4246 unlock: 4247 rcu_read_unlock(); 4248 } 4249 4250 static void mem_cgroup_threshold(struct mem_cgroup *memcg) 4251 { 4252 while (memcg) { 4253 __mem_cgroup_threshold(memcg, false); 4254 if (do_memsw_account()) 4255 __mem_cgroup_threshold(memcg, true); 4256 4257 memcg = parent_mem_cgroup(memcg); 4258 } 4259 } 4260 4261 static int compare_thresholds(const void *a, const void *b) 4262 { 4263 const struct mem_cgroup_threshold *_a = a; 4264 const struct mem_cgroup_threshold *_b = b; 4265 4266 if (_a->threshold > _b->threshold) 4267 return 1; 4268 4269 if (_a->threshold < _b->threshold) 4270 return -1; 4271 4272 return 0; 4273 } 4274 4275 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg) 4276 { 4277 struct mem_cgroup_eventfd_list *ev; 4278 4279 spin_lock(&memcg_oom_lock); 4280 4281 list_for_each_entry(ev, &memcg->oom_notify, list) 4282 eventfd_signal(ev->eventfd, 1); 4283 4284 spin_unlock(&memcg_oom_lock); 4285 return 0; 4286 } 4287 4288 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg) 4289 { 4290 struct mem_cgroup *iter; 4291 4292 for_each_mem_cgroup_tree(iter, memcg) 4293 mem_cgroup_oom_notify_cb(iter); 4294 } 4295 4296 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg, 4297 struct eventfd_ctx *eventfd, const char *args, enum res_type type) 4298 { 4299 struct mem_cgroup_thresholds *thresholds; 4300 struct mem_cgroup_threshold_ary *new; 4301 unsigned long threshold; 4302 unsigned long usage; 4303 int i, size, ret; 4304 4305 ret = page_counter_memparse(args, "-1", &threshold); 4306 if (ret) 4307 return ret; 4308 4309 mutex_lock(&memcg->thresholds_lock); 4310 4311 if (type == _MEM) { 4312 thresholds = &memcg->thresholds; 4313 usage = mem_cgroup_usage(memcg, false); 4314 } else if (type == _MEMSWAP) { 4315 thresholds = &memcg->memsw_thresholds; 4316 usage = mem_cgroup_usage(memcg, true); 4317 } else 4318 BUG(); 4319 4320 /* Check if a threshold crossed before adding a new one */ 4321 if (thresholds->primary) 4322 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 4323 4324 size = thresholds->primary ? thresholds->primary->size + 1 : 1; 4325 4326 /* Allocate memory for new array of thresholds */ 4327 new = kmalloc(struct_size(new, entries, size), GFP_KERNEL); 4328 if (!new) { 4329 ret = -ENOMEM; 4330 goto unlock; 4331 } 4332 new->size = size; 4333 4334 /* Copy thresholds (if any) to new array */ 4335 if (thresholds->primary) 4336 memcpy(new->entries, thresholds->primary->entries, 4337 flex_array_size(new, entries, size - 1)); 4338 4339 /* Add new threshold */ 4340 new->entries[size - 1].eventfd = eventfd; 4341 new->entries[size - 1].threshold = threshold; 4342 4343 /* Sort thresholds. Registering of new threshold isn't time-critical */ 4344 sort(new->entries, size, sizeof(*new->entries), 4345 compare_thresholds, NULL); 4346 4347 /* Find current threshold */ 4348 new->current_threshold = -1; 4349 for (i = 0; i < size; i++) { 4350 if (new->entries[i].threshold <= usage) { 4351 /* 4352 * new->current_threshold will not be used until 4353 * rcu_assign_pointer(), so it's safe to increment 4354 * it here. 4355 */ 4356 ++new->current_threshold; 4357 } else 4358 break; 4359 } 4360 4361 /* Free old spare buffer and save old primary buffer as spare */ 4362 kfree(thresholds->spare); 4363 thresholds->spare = thresholds->primary; 4364 4365 rcu_assign_pointer(thresholds->primary, new); 4366 4367 /* To be sure that nobody uses thresholds */ 4368 synchronize_rcu(); 4369 4370 unlock: 4371 mutex_unlock(&memcg->thresholds_lock); 4372 4373 return ret; 4374 } 4375 4376 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg, 4377 struct eventfd_ctx *eventfd, const char *args) 4378 { 4379 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM); 4380 } 4381 4382 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg, 4383 struct eventfd_ctx *eventfd, const char *args) 4384 { 4385 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP); 4386 } 4387 4388 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, 4389 struct eventfd_ctx *eventfd, enum res_type type) 4390 { 4391 struct mem_cgroup_thresholds *thresholds; 4392 struct mem_cgroup_threshold_ary *new; 4393 unsigned long usage; 4394 int i, j, size, entries; 4395 4396 mutex_lock(&memcg->thresholds_lock); 4397 4398 if (type == _MEM) { 4399 thresholds = &memcg->thresholds; 4400 usage = mem_cgroup_usage(memcg, false); 4401 } else if (type == _MEMSWAP) { 4402 thresholds = &memcg->memsw_thresholds; 4403 usage = mem_cgroup_usage(memcg, true); 4404 } else 4405 BUG(); 4406 4407 if (!thresholds->primary) 4408 goto unlock; 4409 4410 /* Check if a threshold crossed before removing */ 4411 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 4412 4413 /* Calculate new number of threshold */ 4414 size = entries = 0; 4415 for (i = 0; i < thresholds->primary->size; i++) { 4416 if (thresholds->primary->entries[i].eventfd != eventfd) 4417 size++; 4418 else 4419 entries++; 4420 } 4421 4422 new = thresholds->spare; 4423 4424 /* If no items related to eventfd have been cleared, nothing to do */ 4425 if (!entries) 4426 goto unlock; 4427 4428 /* Set thresholds array to NULL if we don't have thresholds */ 4429 if (!size) { 4430 kfree(new); 4431 new = NULL; 4432 goto swap_buffers; 4433 } 4434 4435 new->size = size; 4436 4437 /* Copy thresholds and find current threshold */ 4438 new->current_threshold = -1; 4439 for (i = 0, j = 0; i < thresholds->primary->size; i++) { 4440 if (thresholds->primary->entries[i].eventfd == eventfd) 4441 continue; 4442 4443 new->entries[j] = thresholds->primary->entries[i]; 4444 if (new->entries[j].threshold <= usage) { 4445 /* 4446 * new->current_threshold will not be used 4447 * until rcu_assign_pointer(), so it's safe to increment 4448 * it here. 4449 */ 4450 ++new->current_threshold; 4451 } 4452 j++; 4453 } 4454 4455 swap_buffers: 4456 /* Swap primary and spare array */ 4457 thresholds->spare = thresholds->primary; 4458 4459 rcu_assign_pointer(thresholds->primary, new); 4460 4461 /* To be sure that nobody uses thresholds */ 4462 synchronize_rcu(); 4463 4464 /* If all events are unregistered, free the spare array */ 4465 if (!new) { 4466 kfree(thresholds->spare); 4467 thresholds->spare = NULL; 4468 } 4469 unlock: 4470 mutex_unlock(&memcg->thresholds_lock); 4471 } 4472 4473 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, 4474 struct eventfd_ctx *eventfd) 4475 { 4476 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM); 4477 } 4478 4479 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg, 4480 struct eventfd_ctx *eventfd) 4481 { 4482 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP); 4483 } 4484 4485 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg, 4486 struct eventfd_ctx *eventfd, const char *args) 4487 { 4488 struct mem_cgroup_eventfd_list *event; 4489 4490 event = kmalloc(sizeof(*event), GFP_KERNEL); 4491 if (!event) 4492 return -ENOMEM; 4493 4494 spin_lock(&memcg_oom_lock); 4495 4496 event->eventfd = eventfd; 4497 list_add(&event->list, &memcg->oom_notify); 4498 4499 /* already in OOM ? */ 4500 if (memcg->under_oom) 4501 eventfd_signal(eventfd, 1); 4502 spin_unlock(&memcg_oom_lock); 4503 4504 return 0; 4505 } 4506 4507 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg, 4508 struct eventfd_ctx *eventfd) 4509 { 4510 struct mem_cgroup_eventfd_list *ev, *tmp; 4511 4512 spin_lock(&memcg_oom_lock); 4513 4514 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) { 4515 if (ev->eventfd == eventfd) { 4516 list_del(&ev->list); 4517 kfree(ev); 4518 } 4519 } 4520 4521 spin_unlock(&memcg_oom_lock); 4522 } 4523 4524 static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v) 4525 { 4526 struct mem_cgroup *memcg = mem_cgroup_from_seq(sf); 4527 4528 seq_printf(sf, "oom_kill_disable %d\n", READ_ONCE(memcg->oom_kill_disable)); 4529 seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom); 4530 seq_printf(sf, "oom_kill %lu\n", 4531 atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL])); 4532 return 0; 4533 } 4534 4535 static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css, 4536 struct cftype *cft, u64 val) 4537 { 4538 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4539 4540 /* cannot set to root cgroup and only 0 and 1 are allowed */ 4541 if (mem_cgroup_is_root(memcg) || !((val == 0) || (val == 1))) 4542 return -EINVAL; 4543 4544 WRITE_ONCE(memcg->oom_kill_disable, val); 4545 if (!val) 4546 memcg_oom_recover(memcg); 4547 4548 return 0; 4549 } 4550 4551 #ifdef CONFIG_CGROUP_WRITEBACK 4552 4553 #include <trace/events/writeback.h> 4554 4555 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp) 4556 { 4557 return wb_domain_init(&memcg->cgwb_domain, gfp); 4558 } 4559 4560 static void memcg_wb_domain_exit(struct mem_cgroup *memcg) 4561 { 4562 wb_domain_exit(&memcg->cgwb_domain); 4563 } 4564 4565 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg) 4566 { 4567 wb_domain_size_changed(&memcg->cgwb_domain); 4568 } 4569 4570 struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb) 4571 { 4572 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); 4573 4574 if (!memcg->css.parent) 4575 return NULL; 4576 4577 return &memcg->cgwb_domain; 4578 } 4579 4580 /** 4581 * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg 4582 * @wb: bdi_writeback in question 4583 * @pfilepages: out parameter for number of file pages 4584 * @pheadroom: out parameter for number of allocatable pages according to memcg 4585 * @pdirty: out parameter for number of dirty pages 4586 * @pwriteback: out parameter for number of pages under writeback 4587 * 4588 * Determine the numbers of file, headroom, dirty, and writeback pages in 4589 * @wb's memcg. File, dirty and writeback are self-explanatory. Headroom 4590 * is a bit more involved. 4591 * 4592 * A memcg's headroom is "min(max, high) - used". In the hierarchy, the 4593 * headroom is calculated as the lowest headroom of itself and the 4594 * ancestors. Note that this doesn't consider the actual amount of 4595 * available memory in the system. The caller should further cap 4596 * *@pheadroom accordingly. 4597 */ 4598 void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages, 4599 unsigned long *pheadroom, unsigned long *pdirty, 4600 unsigned long *pwriteback) 4601 { 4602 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); 4603 struct mem_cgroup *parent; 4604 4605 mem_cgroup_flush_stats(); 4606 4607 *pdirty = memcg_page_state(memcg, NR_FILE_DIRTY); 4608 *pwriteback = memcg_page_state(memcg, NR_WRITEBACK); 4609 *pfilepages = memcg_page_state(memcg, NR_INACTIVE_FILE) + 4610 memcg_page_state(memcg, NR_ACTIVE_FILE); 4611 4612 *pheadroom = PAGE_COUNTER_MAX; 4613 while ((parent = parent_mem_cgroup(memcg))) { 4614 unsigned long ceiling = min(READ_ONCE(memcg->memory.max), 4615 READ_ONCE(memcg->memory.high)); 4616 unsigned long used = page_counter_read(&memcg->memory); 4617 4618 *pheadroom = min(*pheadroom, ceiling - min(ceiling, used)); 4619 memcg = parent; 4620 } 4621 } 4622 4623 /* 4624 * Foreign dirty flushing 4625 * 4626 * There's an inherent mismatch between memcg and writeback. The former 4627 * tracks ownership per-page while the latter per-inode. This was a 4628 * deliberate design decision because honoring per-page ownership in the 4629 * writeback path is complicated, may lead to higher CPU and IO overheads 4630 * and deemed unnecessary given that write-sharing an inode across 4631 * different cgroups isn't a common use-case. 4632 * 4633 * Combined with inode majority-writer ownership switching, this works well 4634 * enough in most cases but there are some pathological cases. For 4635 * example, let's say there are two cgroups A and B which keep writing to 4636 * different but confined parts of the same inode. B owns the inode and 4637 * A's memory is limited far below B's. A's dirty ratio can rise enough to 4638 * trigger balance_dirty_pages() sleeps but B's can be low enough to avoid 4639 * triggering background writeback. A will be slowed down without a way to 4640 * make writeback of the dirty pages happen. 4641 * 4642 * Conditions like the above can lead to a cgroup getting repeatedly and 4643 * severely throttled after making some progress after each 4644 * dirty_expire_interval while the underlying IO device is almost 4645 * completely idle. 4646 * 4647 * Solving this problem completely requires matching the ownership tracking 4648 * granularities between memcg and writeback in either direction. However, 4649 * the more egregious behaviors can be avoided by simply remembering the 4650 * most recent foreign dirtying events and initiating remote flushes on 4651 * them when local writeback isn't enough to keep the memory clean enough. 4652 * 4653 * The following two functions implement such mechanism. When a foreign 4654 * page - a page whose memcg and writeback ownerships don't match - is 4655 * dirtied, mem_cgroup_track_foreign_dirty() records the inode owning 4656 * bdi_writeback on the page owning memcg. When balance_dirty_pages() 4657 * decides that the memcg needs to sleep due to high dirty ratio, it calls 4658 * mem_cgroup_flush_foreign() which queues writeback on the recorded 4659 * foreign bdi_writebacks which haven't expired. Both the numbers of 4660 * recorded bdi_writebacks and concurrent in-flight foreign writebacks are 4661 * limited to MEMCG_CGWB_FRN_CNT. 4662 * 4663 * The mechanism only remembers IDs and doesn't hold any object references. 4664 * As being wrong occasionally doesn't matter, updates and accesses to the 4665 * records are lockless and racy. 4666 */ 4667 void mem_cgroup_track_foreign_dirty_slowpath(struct folio *folio, 4668 struct bdi_writeback *wb) 4669 { 4670 struct mem_cgroup *memcg = folio_memcg(folio); 4671 struct memcg_cgwb_frn *frn; 4672 u64 now = get_jiffies_64(); 4673 u64 oldest_at = now; 4674 int oldest = -1; 4675 int i; 4676 4677 trace_track_foreign_dirty(folio, wb); 4678 4679 /* 4680 * Pick the slot to use. If there is already a slot for @wb, keep 4681 * using it. If not replace the oldest one which isn't being 4682 * written out. 4683 */ 4684 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) { 4685 frn = &memcg->cgwb_frn[i]; 4686 if (frn->bdi_id == wb->bdi->id && 4687 frn->memcg_id == wb->memcg_css->id) 4688 break; 4689 if (time_before64(frn->at, oldest_at) && 4690 atomic_read(&frn->done.cnt) == 1) { 4691 oldest = i; 4692 oldest_at = frn->at; 4693 } 4694 } 4695 4696 if (i < MEMCG_CGWB_FRN_CNT) { 4697 /* 4698 * Re-using an existing one. Update timestamp lazily to 4699 * avoid making the cacheline hot. We want them to be 4700 * reasonably up-to-date and significantly shorter than 4701 * dirty_expire_interval as that's what expires the record. 4702 * Use the shorter of 1s and dirty_expire_interval / 8. 4703 */ 4704 unsigned long update_intv = 4705 min_t(unsigned long, HZ, 4706 msecs_to_jiffies(dirty_expire_interval * 10) / 8); 4707 4708 if (time_before64(frn->at, now - update_intv)) 4709 frn->at = now; 4710 } else if (oldest >= 0) { 4711 /* replace the oldest free one */ 4712 frn = &memcg->cgwb_frn[oldest]; 4713 frn->bdi_id = wb->bdi->id; 4714 frn->memcg_id = wb->memcg_css->id; 4715 frn->at = now; 4716 } 4717 } 4718 4719 /* issue foreign writeback flushes for recorded foreign dirtying events */ 4720 void mem_cgroup_flush_foreign(struct bdi_writeback *wb) 4721 { 4722 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); 4723 unsigned long intv = msecs_to_jiffies(dirty_expire_interval * 10); 4724 u64 now = jiffies_64; 4725 int i; 4726 4727 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) { 4728 struct memcg_cgwb_frn *frn = &memcg->cgwb_frn[i]; 4729 4730 /* 4731 * If the record is older than dirty_expire_interval, 4732 * writeback on it has already started. No need to kick it 4733 * off again. Also, don't start a new one if there's 4734 * already one in flight. 4735 */ 4736 if (time_after64(frn->at, now - intv) && 4737 atomic_read(&frn->done.cnt) == 1) { 4738 frn->at = 0; 4739 trace_flush_foreign(wb, frn->bdi_id, frn->memcg_id); 4740 cgroup_writeback_by_id(frn->bdi_id, frn->memcg_id, 4741 WB_REASON_FOREIGN_FLUSH, 4742 &frn->done); 4743 } 4744 } 4745 } 4746 4747 #else /* CONFIG_CGROUP_WRITEBACK */ 4748 4749 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp) 4750 { 4751 return 0; 4752 } 4753 4754 static void memcg_wb_domain_exit(struct mem_cgroup *memcg) 4755 { 4756 } 4757 4758 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg) 4759 { 4760 } 4761 4762 #endif /* CONFIG_CGROUP_WRITEBACK */ 4763 4764 /* 4765 * DO NOT USE IN NEW FILES. 4766 * 4767 * "cgroup.event_control" implementation. 4768 * 4769 * This is way over-engineered. It tries to support fully configurable 4770 * events for each user. Such level of flexibility is completely 4771 * unnecessary especially in the light of the planned unified hierarchy. 4772 * 4773 * Please deprecate this and replace with something simpler if at all 4774 * possible. 4775 */ 4776 4777 /* 4778 * Unregister event and free resources. 4779 * 4780 * Gets called from workqueue. 4781 */ 4782 static void memcg_event_remove(struct work_struct *work) 4783 { 4784 struct mem_cgroup_event *event = 4785 container_of(work, struct mem_cgroup_event, remove); 4786 struct mem_cgroup *memcg = event->memcg; 4787 4788 remove_wait_queue(event->wqh, &event->wait); 4789 4790 event->unregister_event(memcg, event->eventfd); 4791 4792 /* Notify userspace the event is going away. */ 4793 eventfd_signal(event->eventfd, 1); 4794 4795 eventfd_ctx_put(event->eventfd); 4796 kfree(event); 4797 css_put(&memcg->css); 4798 } 4799 4800 /* 4801 * Gets called on EPOLLHUP on eventfd when user closes it. 4802 * 4803 * Called with wqh->lock held and interrupts disabled. 4804 */ 4805 static int memcg_event_wake(wait_queue_entry_t *wait, unsigned mode, 4806 int sync, void *key) 4807 { 4808 struct mem_cgroup_event *event = 4809 container_of(wait, struct mem_cgroup_event, wait); 4810 struct mem_cgroup *memcg = event->memcg; 4811 __poll_t flags = key_to_poll(key); 4812 4813 if (flags & EPOLLHUP) { 4814 /* 4815 * If the event has been detached at cgroup removal, we 4816 * can simply return knowing the other side will cleanup 4817 * for us. 4818 * 4819 * We can't race against event freeing since the other 4820 * side will require wqh->lock via remove_wait_queue(), 4821 * which we hold. 4822 */ 4823 spin_lock(&memcg->event_list_lock); 4824 if (!list_empty(&event->list)) { 4825 list_del_init(&event->list); 4826 /* 4827 * We are in atomic context, but cgroup_event_remove() 4828 * may sleep, so we have to call it in workqueue. 4829 */ 4830 schedule_work(&event->remove); 4831 } 4832 spin_unlock(&memcg->event_list_lock); 4833 } 4834 4835 return 0; 4836 } 4837 4838 static void memcg_event_ptable_queue_proc(struct file *file, 4839 wait_queue_head_t *wqh, poll_table *pt) 4840 { 4841 struct mem_cgroup_event *event = 4842 container_of(pt, struct mem_cgroup_event, pt); 4843 4844 event->wqh = wqh; 4845 add_wait_queue(wqh, &event->wait); 4846 } 4847 4848 /* 4849 * DO NOT USE IN NEW FILES. 4850 * 4851 * Parse input and register new cgroup event handler. 4852 * 4853 * Input must be in format '<event_fd> <control_fd> <args>'. 4854 * Interpretation of args is defined by control file implementation. 4855 */ 4856 static ssize_t memcg_write_event_control(struct kernfs_open_file *of, 4857 char *buf, size_t nbytes, loff_t off) 4858 { 4859 struct cgroup_subsys_state *css = of_css(of); 4860 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4861 struct mem_cgroup_event *event; 4862 struct cgroup_subsys_state *cfile_css; 4863 unsigned int efd, cfd; 4864 struct fd efile; 4865 struct fd cfile; 4866 struct dentry *cdentry; 4867 const char *name; 4868 char *endp; 4869 int ret; 4870 4871 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 4872 return -EOPNOTSUPP; 4873 4874 buf = strstrip(buf); 4875 4876 efd = simple_strtoul(buf, &endp, 10); 4877 if (*endp != ' ') 4878 return -EINVAL; 4879 buf = endp + 1; 4880 4881 cfd = simple_strtoul(buf, &endp, 10); 4882 if ((*endp != ' ') && (*endp != '\0')) 4883 return -EINVAL; 4884 buf = endp + 1; 4885 4886 event = kzalloc(sizeof(*event), GFP_KERNEL); 4887 if (!event) 4888 return -ENOMEM; 4889 4890 event->memcg = memcg; 4891 INIT_LIST_HEAD(&event->list); 4892 init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc); 4893 init_waitqueue_func_entry(&event->wait, memcg_event_wake); 4894 INIT_WORK(&event->remove, memcg_event_remove); 4895 4896 efile = fdget(efd); 4897 if (!efile.file) { 4898 ret = -EBADF; 4899 goto out_kfree; 4900 } 4901 4902 event->eventfd = eventfd_ctx_fileget(efile.file); 4903 if (IS_ERR(event->eventfd)) { 4904 ret = PTR_ERR(event->eventfd); 4905 goto out_put_efile; 4906 } 4907 4908 cfile = fdget(cfd); 4909 if (!cfile.file) { 4910 ret = -EBADF; 4911 goto out_put_eventfd; 4912 } 4913 4914 /* the process need read permission on control file */ 4915 /* AV: shouldn't we check that it's been opened for read instead? */ 4916 ret = file_permission(cfile.file, MAY_READ); 4917 if (ret < 0) 4918 goto out_put_cfile; 4919 4920 /* 4921 * The control file must be a regular cgroup1 file. As a regular cgroup 4922 * file can't be renamed, it's safe to access its name afterwards. 4923 */ 4924 cdentry = cfile.file->f_path.dentry; 4925 if (cdentry->d_sb->s_type != &cgroup_fs_type || !d_is_reg(cdentry)) { 4926 ret = -EINVAL; 4927 goto out_put_cfile; 4928 } 4929 4930 /* 4931 * Determine the event callbacks and set them in @event. This used 4932 * to be done via struct cftype but cgroup core no longer knows 4933 * about these events. The following is crude but the whole thing 4934 * is for compatibility anyway. 4935 * 4936 * DO NOT ADD NEW FILES. 4937 */ 4938 name = cdentry->d_name.name; 4939 4940 if (!strcmp(name, "memory.usage_in_bytes")) { 4941 event->register_event = mem_cgroup_usage_register_event; 4942 event->unregister_event = mem_cgroup_usage_unregister_event; 4943 } else if (!strcmp(name, "memory.oom_control")) { 4944 event->register_event = mem_cgroup_oom_register_event; 4945 event->unregister_event = mem_cgroup_oom_unregister_event; 4946 } else if (!strcmp(name, "memory.pressure_level")) { 4947 event->register_event = vmpressure_register_event; 4948 event->unregister_event = vmpressure_unregister_event; 4949 } else if (!strcmp(name, "memory.memsw.usage_in_bytes")) { 4950 event->register_event = memsw_cgroup_usage_register_event; 4951 event->unregister_event = memsw_cgroup_usage_unregister_event; 4952 } else { 4953 ret = -EINVAL; 4954 goto out_put_cfile; 4955 } 4956 4957 /* 4958 * Verify @cfile should belong to @css. Also, remaining events are 4959 * automatically removed on cgroup destruction but the removal is 4960 * asynchronous, so take an extra ref on @css. 4961 */ 4962 cfile_css = css_tryget_online_from_dir(cdentry->d_parent, 4963 &memory_cgrp_subsys); 4964 ret = -EINVAL; 4965 if (IS_ERR(cfile_css)) 4966 goto out_put_cfile; 4967 if (cfile_css != css) { 4968 css_put(cfile_css); 4969 goto out_put_cfile; 4970 } 4971 4972 ret = event->register_event(memcg, event->eventfd, buf); 4973 if (ret) 4974 goto out_put_css; 4975 4976 vfs_poll(efile.file, &event->pt); 4977 4978 spin_lock_irq(&memcg->event_list_lock); 4979 list_add(&event->list, &memcg->event_list); 4980 spin_unlock_irq(&memcg->event_list_lock); 4981 4982 fdput(cfile); 4983 fdput(efile); 4984 4985 return nbytes; 4986 4987 out_put_css: 4988 css_put(css); 4989 out_put_cfile: 4990 fdput(cfile); 4991 out_put_eventfd: 4992 eventfd_ctx_put(event->eventfd); 4993 out_put_efile: 4994 fdput(efile); 4995 out_kfree: 4996 kfree(event); 4997 4998 return ret; 4999 } 5000 5001 #if defined(CONFIG_MEMCG_KMEM) && (defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)) 5002 static int mem_cgroup_slab_show(struct seq_file *m, void *p) 5003 { 5004 /* 5005 * Deprecated. 5006 * Please, take a look at tools/cgroup/memcg_slabinfo.py . 5007 */ 5008 return 0; 5009 } 5010 #endif 5011 5012 static int memory_stat_show(struct seq_file *m, void *v); 5013 5014 static struct cftype mem_cgroup_legacy_files[] = { 5015 { 5016 .name = "usage_in_bytes", 5017 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE), 5018 .read_u64 = mem_cgroup_read_u64, 5019 }, 5020 { 5021 .name = "max_usage_in_bytes", 5022 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE), 5023 .write = mem_cgroup_reset, 5024 .read_u64 = mem_cgroup_read_u64, 5025 }, 5026 { 5027 .name = "limit_in_bytes", 5028 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT), 5029 .write = mem_cgroup_write, 5030 .read_u64 = mem_cgroup_read_u64, 5031 }, 5032 { 5033 .name = "soft_limit_in_bytes", 5034 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT), 5035 .write = mem_cgroup_write, 5036 .read_u64 = mem_cgroup_read_u64, 5037 }, 5038 { 5039 .name = "failcnt", 5040 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT), 5041 .write = mem_cgroup_reset, 5042 .read_u64 = mem_cgroup_read_u64, 5043 }, 5044 { 5045 .name = "stat", 5046 .seq_show = memory_stat_show, 5047 }, 5048 { 5049 .name = "force_empty", 5050 .write = mem_cgroup_force_empty_write, 5051 }, 5052 { 5053 .name = "use_hierarchy", 5054 .write_u64 = mem_cgroup_hierarchy_write, 5055 .read_u64 = mem_cgroup_hierarchy_read, 5056 }, 5057 { 5058 .name = "cgroup.event_control", /* XXX: for compat */ 5059 .write = memcg_write_event_control, 5060 .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE, 5061 }, 5062 { 5063 .name = "swappiness", 5064 .read_u64 = mem_cgroup_swappiness_read, 5065 .write_u64 = mem_cgroup_swappiness_write, 5066 }, 5067 { 5068 .name = "move_charge_at_immigrate", 5069 .read_u64 = mem_cgroup_move_charge_read, 5070 .write_u64 = mem_cgroup_move_charge_write, 5071 }, 5072 { 5073 .name = "oom_control", 5074 .seq_show = mem_cgroup_oom_control_read, 5075 .write_u64 = mem_cgroup_oom_control_write, 5076 }, 5077 { 5078 .name = "pressure_level", 5079 .seq_show = mem_cgroup_dummy_seq_show, 5080 }, 5081 #ifdef CONFIG_NUMA 5082 { 5083 .name = "numa_stat", 5084 .seq_show = memcg_numa_stat_show, 5085 }, 5086 #endif 5087 { 5088 .name = "kmem.limit_in_bytes", 5089 .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT), 5090 .write = mem_cgroup_write, 5091 .read_u64 = mem_cgroup_read_u64, 5092 }, 5093 { 5094 .name = "kmem.usage_in_bytes", 5095 .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE), 5096 .read_u64 = mem_cgroup_read_u64, 5097 }, 5098 { 5099 .name = "kmem.failcnt", 5100 .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT), 5101 .write = mem_cgroup_reset, 5102 .read_u64 = mem_cgroup_read_u64, 5103 }, 5104 { 5105 .name = "kmem.max_usage_in_bytes", 5106 .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE), 5107 .write = mem_cgroup_reset, 5108 .read_u64 = mem_cgroup_read_u64, 5109 }, 5110 #if defined(CONFIG_MEMCG_KMEM) && \ 5111 (defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)) 5112 { 5113 .name = "kmem.slabinfo", 5114 .seq_show = mem_cgroup_slab_show, 5115 }, 5116 #endif 5117 { 5118 .name = "kmem.tcp.limit_in_bytes", 5119 .private = MEMFILE_PRIVATE(_TCP, RES_LIMIT), 5120 .write = mem_cgroup_write, 5121 .read_u64 = mem_cgroup_read_u64, 5122 }, 5123 { 5124 .name = "kmem.tcp.usage_in_bytes", 5125 .private = MEMFILE_PRIVATE(_TCP, RES_USAGE), 5126 .read_u64 = mem_cgroup_read_u64, 5127 }, 5128 { 5129 .name = "kmem.tcp.failcnt", 5130 .private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT), 5131 .write = mem_cgroup_reset, 5132 .read_u64 = mem_cgroup_read_u64, 5133 }, 5134 { 5135 .name = "kmem.tcp.max_usage_in_bytes", 5136 .private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE), 5137 .write = mem_cgroup_reset, 5138 .read_u64 = mem_cgroup_read_u64, 5139 }, 5140 { }, /* terminate */ 5141 }; 5142 5143 /* 5144 * Private memory cgroup IDR 5145 * 5146 * Swap-out records and page cache shadow entries need to store memcg 5147 * references in constrained space, so we maintain an ID space that is 5148 * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of 5149 * memory-controlled cgroups to 64k. 5150 * 5151 * However, there usually are many references to the offline CSS after 5152 * the cgroup has been destroyed, such as page cache or reclaimable 5153 * slab objects, that don't need to hang on to the ID. We want to keep 5154 * those dead CSS from occupying IDs, or we might quickly exhaust the 5155 * relatively small ID space and prevent the creation of new cgroups 5156 * even when there are much fewer than 64k cgroups - possibly none. 5157 * 5158 * Maintain a private 16-bit ID space for memcg, and allow the ID to 5159 * be freed and recycled when it's no longer needed, which is usually 5160 * when the CSS is offlined. 5161 * 5162 * The only exception to that are records of swapped out tmpfs/shmem 5163 * pages that need to be attributed to live ancestors on swapin. But 5164 * those references are manageable from userspace. 5165 */ 5166 5167 #define MEM_CGROUP_ID_MAX ((1UL << MEM_CGROUP_ID_SHIFT) - 1) 5168 static DEFINE_IDR(mem_cgroup_idr); 5169 5170 static void mem_cgroup_id_remove(struct mem_cgroup *memcg) 5171 { 5172 if (memcg->id.id > 0) { 5173 idr_remove(&mem_cgroup_idr, memcg->id.id); 5174 memcg->id.id = 0; 5175 } 5176 } 5177 5178 static void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg, 5179 unsigned int n) 5180 { 5181 refcount_add(n, &memcg->id.ref); 5182 } 5183 5184 static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n) 5185 { 5186 if (refcount_sub_and_test(n, &memcg->id.ref)) { 5187 mem_cgroup_id_remove(memcg); 5188 5189 /* Memcg ID pins CSS */ 5190 css_put(&memcg->css); 5191 } 5192 } 5193 5194 static inline void mem_cgroup_id_put(struct mem_cgroup *memcg) 5195 { 5196 mem_cgroup_id_put_many(memcg, 1); 5197 } 5198 5199 /** 5200 * mem_cgroup_from_id - look up a memcg from a memcg id 5201 * @id: the memcg id to look up 5202 * 5203 * Caller must hold rcu_read_lock(). 5204 */ 5205 struct mem_cgroup *mem_cgroup_from_id(unsigned short id) 5206 { 5207 WARN_ON_ONCE(!rcu_read_lock_held()); 5208 return idr_find(&mem_cgroup_idr, id); 5209 } 5210 5211 #ifdef CONFIG_SHRINKER_DEBUG 5212 struct mem_cgroup *mem_cgroup_get_from_ino(unsigned long ino) 5213 { 5214 struct cgroup *cgrp; 5215 struct cgroup_subsys_state *css; 5216 struct mem_cgroup *memcg; 5217 5218 cgrp = cgroup_get_from_id(ino); 5219 if (IS_ERR(cgrp)) 5220 return ERR_CAST(cgrp); 5221 5222 css = cgroup_get_e_css(cgrp, &memory_cgrp_subsys); 5223 if (css) 5224 memcg = container_of(css, struct mem_cgroup, css); 5225 else 5226 memcg = ERR_PTR(-ENOENT); 5227 5228 cgroup_put(cgrp); 5229 5230 return memcg; 5231 } 5232 #endif 5233 5234 static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) 5235 { 5236 struct mem_cgroup_per_node *pn; 5237 5238 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, node); 5239 if (!pn) 5240 return 1; 5241 5242 pn->lruvec_stats_percpu = alloc_percpu_gfp(struct lruvec_stats_percpu, 5243 GFP_KERNEL_ACCOUNT); 5244 if (!pn->lruvec_stats_percpu) { 5245 kfree(pn); 5246 return 1; 5247 } 5248 5249 lruvec_init(&pn->lruvec); 5250 pn->memcg = memcg; 5251 5252 memcg->nodeinfo[node] = pn; 5253 return 0; 5254 } 5255 5256 static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) 5257 { 5258 struct mem_cgroup_per_node *pn = memcg->nodeinfo[node]; 5259 5260 if (!pn) 5261 return; 5262 5263 free_percpu(pn->lruvec_stats_percpu); 5264 kfree(pn); 5265 } 5266 5267 static void __mem_cgroup_free(struct mem_cgroup *memcg) 5268 { 5269 int node; 5270 5271 for_each_node(node) 5272 free_mem_cgroup_per_node_info(memcg, node); 5273 kfree(memcg->vmstats); 5274 free_percpu(memcg->vmstats_percpu); 5275 kfree(memcg); 5276 } 5277 5278 static void mem_cgroup_free(struct mem_cgroup *memcg) 5279 { 5280 lru_gen_exit_memcg(memcg); 5281 memcg_wb_domain_exit(memcg); 5282 __mem_cgroup_free(memcg); 5283 } 5284 5285 static struct mem_cgroup *mem_cgroup_alloc(void) 5286 { 5287 struct mem_cgroup *memcg; 5288 int node; 5289 int __maybe_unused i; 5290 long error = -ENOMEM; 5291 5292 memcg = kzalloc(struct_size(memcg, nodeinfo, nr_node_ids), GFP_KERNEL); 5293 if (!memcg) 5294 return ERR_PTR(error); 5295 5296 memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL, 5297 1, MEM_CGROUP_ID_MAX + 1, GFP_KERNEL); 5298 if (memcg->id.id < 0) { 5299 error = memcg->id.id; 5300 goto fail; 5301 } 5302 5303 memcg->vmstats = kzalloc(sizeof(struct memcg_vmstats), GFP_KERNEL); 5304 if (!memcg->vmstats) 5305 goto fail; 5306 5307 memcg->vmstats_percpu = alloc_percpu_gfp(struct memcg_vmstats_percpu, 5308 GFP_KERNEL_ACCOUNT); 5309 if (!memcg->vmstats_percpu) 5310 goto fail; 5311 5312 for_each_node(node) 5313 if (alloc_mem_cgroup_per_node_info(memcg, node)) 5314 goto fail; 5315 5316 if (memcg_wb_domain_init(memcg, GFP_KERNEL)) 5317 goto fail; 5318 5319 INIT_WORK(&memcg->high_work, high_work_func); 5320 INIT_LIST_HEAD(&memcg->oom_notify); 5321 mutex_init(&memcg->thresholds_lock); 5322 spin_lock_init(&memcg->move_lock); 5323 vmpressure_init(&memcg->vmpressure); 5324 INIT_LIST_HEAD(&memcg->event_list); 5325 spin_lock_init(&memcg->event_list_lock); 5326 memcg->socket_pressure = jiffies; 5327 #ifdef CONFIG_MEMCG_KMEM 5328 memcg->kmemcg_id = -1; 5329 INIT_LIST_HEAD(&memcg->objcg_list); 5330 #endif 5331 #ifdef CONFIG_CGROUP_WRITEBACK 5332 INIT_LIST_HEAD(&memcg->cgwb_list); 5333 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) 5334 memcg->cgwb_frn[i].done = 5335 __WB_COMPLETION_INIT(&memcg_cgwb_frn_waitq); 5336 #endif 5337 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5338 spin_lock_init(&memcg->deferred_split_queue.split_queue_lock); 5339 INIT_LIST_HEAD(&memcg->deferred_split_queue.split_queue); 5340 memcg->deferred_split_queue.split_queue_len = 0; 5341 #endif 5342 lru_gen_init_memcg(memcg); 5343 return memcg; 5344 fail: 5345 mem_cgroup_id_remove(memcg); 5346 __mem_cgroup_free(memcg); 5347 return ERR_PTR(error); 5348 } 5349 5350 static struct cgroup_subsys_state * __ref 5351 mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 5352 { 5353 struct mem_cgroup *parent = mem_cgroup_from_css(parent_css); 5354 struct mem_cgroup *memcg, *old_memcg; 5355 5356 old_memcg = set_active_memcg(parent); 5357 memcg = mem_cgroup_alloc(); 5358 set_active_memcg(old_memcg); 5359 if (IS_ERR(memcg)) 5360 return ERR_CAST(memcg); 5361 5362 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX); 5363 WRITE_ONCE(memcg->soft_limit, PAGE_COUNTER_MAX); 5364 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP) 5365 memcg->zswap_max = PAGE_COUNTER_MAX; 5366 #endif 5367 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX); 5368 if (parent) { 5369 WRITE_ONCE(memcg->swappiness, mem_cgroup_swappiness(parent)); 5370 WRITE_ONCE(memcg->oom_kill_disable, READ_ONCE(parent->oom_kill_disable)); 5371 5372 page_counter_init(&memcg->memory, &parent->memory); 5373 page_counter_init(&memcg->swap, &parent->swap); 5374 page_counter_init(&memcg->kmem, &parent->kmem); 5375 page_counter_init(&memcg->tcpmem, &parent->tcpmem); 5376 } else { 5377 init_memcg_events(); 5378 page_counter_init(&memcg->memory, NULL); 5379 page_counter_init(&memcg->swap, NULL); 5380 page_counter_init(&memcg->kmem, NULL); 5381 page_counter_init(&memcg->tcpmem, NULL); 5382 5383 root_mem_cgroup = memcg; 5384 return &memcg->css; 5385 } 5386 5387 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket) 5388 static_branch_inc(&memcg_sockets_enabled_key); 5389 5390 #if defined(CONFIG_MEMCG_KMEM) 5391 if (!cgroup_memory_nobpf) 5392 static_branch_inc(&memcg_bpf_enabled_key); 5393 #endif 5394 5395 return &memcg->css; 5396 } 5397 5398 static int mem_cgroup_css_online(struct cgroup_subsys_state *css) 5399 { 5400 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5401 5402 if (memcg_online_kmem(memcg)) 5403 goto remove_id; 5404 5405 /* 5406 * A memcg must be visible for expand_shrinker_info() 5407 * by the time the maps are allocated. So, we allocate maps 5408 * here, when for_each_mem_cgroup() can't skip it. 5409 */ 5410 if (alloc_shrinker_info(memcg)) 5411 goto offline_kmem; 5412 5413 if (unlikely(mem_cgroup_is_root(memcg))) 5414 queue_delayed_work(system_unbound_wq, &stats_flush_dwork, 5415 FLUSH_TIME); 5416 lru_gen_online_memcg(memcg); 5417 5418 /* Online state pins memcg ID, memcg ID pins CSS */ 5419 refcount_set(&memcg->id.ref, 1); 5420 css_get(css); 5421 5422 /* 5423 * Ensure mem_cgroup_from_id() works once we're fully online. 5424 * 5425 * We could do this earlier and require callers to filter with 5426 * css_tryget_online(). But right now there are no users that 5427 * need earlier access, and the workingset code relies on the 5428 * cgroup tree linkage (mem_cgroup_get_nr_swap_pages()). So 5429 * publish it here at the end of onlining. This matches the 5430 * regular ID destruction during offlining. 5431 */ 5432 idr_replace(&mem_cgroup_idr, memcg, memcg->id.id); 5433 5434 return 0; 5435 offline_kmem: 5436 memcg_offline_kmem(memcg); 5437 remove_id: 5438 mem_cgroup_id_remove(memcg); 5439 return -ENOMEM; 5440 } 5441 5442 static void mem_cgroup_css_offline(struct cgroup_subsys_state *css) 5443 { 5444 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5445 struct mem_cgroup_event *event, *tmp; 5446 5447 /* 5448 * Unregister events and notify userspace. 5449 * Notify userspace about cgroup removing only after rmdir of cgroup 5450 * directory to avoid race between userspace and kernelspace. 5451 */ 5452 spin_lock_irq(&memcg->event_list_lock); 5453 list_for_each_entry_safe(event, tmp, &memcg->event_list, list) { 5454 list_del_init(&event->list); 5455 schedule_work(&event->remove); 5456 } 5457 spin_unlock_irq(&memcg->event_list_lock); 5458 5459 page_counter_set_min(&memcg->memory, 0); 5460 page_counter_set_low(&memcg->memory, 0); 5461 5462 memcg_offline_kmem(memcg); 5463 reparent_shrinker_deferred(memcg); 5464 wb_memcg_offline(memcg); 5465 lru_gen_offline_memcg(memcg); 5466 5467 drain_all_stock(memcg); 5468 5469 mem_cgroup_id_put(memcg); 5470 } 5471 5472 static void mem_cgroup_css_released(struct cgroup_subsys_state *css) 5473 { 5474 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5475 5476 invalidate_reclaim_iterators(memcg); 5477 lru_gen_release_memcg(memcg); 5478 } 5479 5480 static void mem_cgroup_css_free(struct cgroup_subsys_state *css) 5481 { 5482 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5483 int __maybe_unused i; 5484 5485 #ifdef CONFIG_CGROUP_WRITEBACK 5486 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) 5487 wb_wait_for_completion(&memcg->cgwb_frn[i].done); 5488 #endif 5489 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket) 5490 static_branch_dec(&memcg_sockets_enabled_key); 5491 5492 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active) 5493 static_branch_dec(&memcg_sockets_enabled_key); 5494 5495 #if defined(CONFIG_MEMCG_KMEM) 5496 if (!cgroup_memory_nobpf) 5497 static_branch_dec(&memcg_bpf_enabled_key); 5498 #endif 5499 5500 vmpressure_cleanup(&memcg->vmpressure); 5501 cancel_work_sync(&memcg->high_work); 5502 mem_cgroup_remove_from_trees(memcg); 5503 free_shrinker_info(memcg); 5504 mem_cgroup_free(memcg); 5505 } 5506 5507 /** 5508 * mem_cgroup_css_reset - reset the states of a mem_cgroup 5509 * @css: the target css 5510 * 5511 * Reset the states of the mem_cgroup associated with @css. This is 5512 * invoked when the userland requests disabling on the default hierarchy 5513 * but the memcg is pinned through dependency. The memcg should stop 5514 * applying policies and should revert to the vanilla state as it may be 5515 * made visible again. 5516 * 5517 * The current implementation only resets the essential configurations. 5518 * This needs to be expanded to cover all the visible parts. 5519 */ 5520 static void mem_cgroup_css_reset(struct cgroup_subsys_state *css) 5521 { 5522 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5523 5524 page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX); 5525 page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX); 5526 page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX); 5527 page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX); 5528 page_counter_set_min(&memcg->memory, 0); 5529 page_counter_set_low(&memcg->memory, 0); 5530 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX); 5531 WRITE_ONCE(memcg->soft_limit, PAGE_COUNTER_MAX); 5532 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX); 5533 memcg_wb_domain_size_changed(memcg); 5534 } 5535 5536 static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu) 5537 { 5538 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5539 struct mem_cgroup *parent = parent_mem_cgroup(memcg); 5540 struct memcg_vmstats_percpu *statc; 5541 long delta, delta_cpu, v; 5542 int i, nid; 5543 5544 statc = per_cpu_ptr(memcg->vmstats_percpu, cpu); 5545 5546 for (i = 0; i < MEMCG_NR_STAT; i++) { 5547 /* 5548 * Collect the aggregated propagation counts of groups 5549 * below us. We're in a per-cpu loop here and this is 5550 * a global counter, so the first cycle will get them. 5551 */ 5552 delta = memcg->vmstats->state_pending[i]; 5553 if (delta) 5554 memcg->vmstats->state_pending[i] = 0; 5555 5556 /* Add CPU changes on this level since the last flush */ 5557 delta_cpu = 0; 5558 v = READ_ONCE(statc->state[i]); 5559 if (v != statc->state_prev[i]) { 5560 delta_cpu = v - statc->state_prev[i]; 5561 delta += delta_cpu; 5562 statc->state_prev[i] = v; 5563 } 5564 5565 /* Aggregate counts on this level and propagate upwards */ 5566 if (delta_cpu) 5567 memcg->vmstats->state_local[i] += delta_cpu; 5568 5569 if (delta) { 5570 memcg->vmstats->state[i] += delta; 5571 if (parent) 5572 parent->vmstats->state_pending[i] += delta; 5573 } 5574 } 5575 5576 for (i = 0; i < NR_MEMCG_EVENTS; i++) { 5577 delta = memcg->vmstats->events_pending[i]; 5578 if (delta) 5579 memcg->vmstats->events_pending[i] = 0; 5580 5581 delta_cpu = 0; 5582 v = READ_ONCE(statc->events[i]); 5583 if (v != statc->events_prev[i]) { 5584 delta_cpu = v - statc->events_prev[i]; 5585 delta += delta_cpu; 5586 statc->events_prev[i] = v; 5587 } 5588 5589 if (delta_cpu) 5590 memcg->vmstats->events_local[i] += delta_cpu; 5591 5592 if (delta) { 5593 memcg->vmstats->events[i] += delta; 5594 if (parent) 5595 parent->vmstats->events_pending[i] += delta; 5596 } 5597 } 5598 5599 for_each_node_state(nid, N_MEMORY) { 5600 struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid]; 5601 struct mem_cgroup_per_node *ppn = NULL; 5602 struct lruvec_stats_percpu *lstatc; 5603 5604 if (parent) 5605 ppn = parent->nodeinfo[nid]; 5606 5607 lstatc = per_cpu_ptr(pn->lruvec_stats_percpu, cpu); 5608 5609 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 5610 delta = pn->lruvec_stats.state_pending[i]; 5611 if (delta) 5612 pn->lruvec_stats.state_pending[i] = 0; 5613 5614 delta_cpu = 0; 5615 v = READ_ONCE(lstatc->state[i]); 5616 if (v != lstatc->state_prev[i]) { 5617 delta_cpu = v - lstatc->state_prev[i]; 5618 delta += delta_cpu; 5619 lstatc->state_prev[i] = v; 5620 } 5621 5622 if (delta_cpu) 5623 pn->lruvec_stats.state_local[i] += delta_cpu; 5624 5625 if (delta) { 5626 pn->lruvec_stats.state[i] += delta; 5627 if (ppn) 5628 ppn->lruvec_stats.state_pending[i] += delta; 5629 } 5630 } 5631 } 5632 } 5633 5634 #ifdef CONFIG_MMU 5635 /* Handlers for move charge at task migration. */ 5636 static int mem_cgroup_do_precharge(unsigned long count) 5637 { 5638 int ret; 5639 5640 /* Try a single bulk charge without reclaim first, kswapd may wake */ 5641 ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count); 5642 if (!ret) { 5643 mc.precharge += count; 5644 return ret; 5645 } 5646 5647 /* Try charges one by one with reclaim, but do not retry */ 5648 while (count--) { 5649 ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1); 5650 if (ret) 5651 return ret; 5652 mc.precharge++; 5653 cond_resched(); 5654 } 5655 return 0; 5656 } 5657 5658 union mc_target { 5659 struct page *page; 5660 swp_entry_t ent; 5661 }; 5662 5663 enum mc_target_type { 5664 MC_TARGET_NONE = 0, 5665 MC_TARGET_PAGE, 5666 MC_TARGET_SWAP, 5667 MC_TARGET_DEVICE, 5668 }; 5669 5670 static struct page *mc_handle_present_pte(struct vm_area_struct *vma, 5671 unsigned long addr, pte_t ptent) 5672 { 5673 struct page *page = vm_normal_page(vma, addr, ptent); 5674 5675 if (!page) 5676 return NULL; 5677 if (PageAnon(page)) { 5678 if (!(mc.flags & MOVE_ANON)) 5679 return NULL; 5680 } else { 5681 if (!(mc.flags & MOVE_FILE)) 5682 return NULL; 5683 } 5684 get_page(page); 5685 5686 return page; 5687 } 5688 5689 #if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE) 5690 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma, 5691 pte_t ptent, swp_entry_t *entry) 5692 { 5693 struct page *page = NULL; 5694 swp_entry_t ent = pte_to_swp_entry(ptent); 5695 5696 if (!(mc.flags & MOVE_ANON)) 5697 return NULL; 5698 5699 /* 5700 * Handle device private pages that are not accessible by the CPU, but 5701 * stored as special swap entries in the page table. 5702 */ 5703 if (is_device_private_entry(ent)) { 5704 page = pfn_swap_entry_to_page(ent); 5705 if (!get_page_unless_zero(page)) 5706 return NULL; 5707 return page; 5708 } 5709 5710 if (non_swap_entry(ent)) 5711 return NULL; 5712 5713 /* 5714 * Because swap_cache_get_folio() updates some statistics counter, 5715 * we call find_get_page() with swapper_space directly. 5716 */ 5717 page = find_get_page(swap_address_space(ent), swp_offset(ent)); 5718 entry->val = ent.val; 5719 5720 return page; 5721 } 5722 #else 5723 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma, 5724 pte_t ptent, swp_entry_t *entry) 5725 { 5726 return NULL; 5727 } 5728 #endif 5729 5730 static struct page *mc_handle_file_pte(struct vm_area_struct *vma, 5731 unsigned long addr, pte_t ptent) 5732 { 5733 unsigned long index; 5734 struct folio *folio; 5735 5736 if (!vma->vm_file) /* anonymous vma */ 5737 return NULL; 5738 if (!(mc.flags & MOVE_FILE)) 5739 return NULL; 5740 5741 /* folio is moved even if it's not RSS of this task(page-faulted). */ 5742 /* shmem/tmpfs may report page out on swap: account for that too. */ 5743 index = linear_page_index(vma, addr); 5744 folio = filemap_get_incore_folio(vma->vm_file->f_mapping, index); 5745 if (IS_ERR(folio)) 5746 return NULL; 5747 return folio_file_page(folio, index); 5748 } 5749 5750 /** 5751 * mem_cgroup_move_account - move account of the page 5752 * @page: the page 5753 * @compound: charge the page as compound or small page 5754 * @from: mem_cgroup which the page is moved from. 5755 * @to: mem_cgroup which the page is moved to. @from != @to. 5756 * 5757 * The page must be locked and not on the LRU. 5758 * 5759 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge" 5760 * from old cgroup. 5761 */ 5762 static int mem_cgroup_move_account(struct page *page, 5763 bool compound, 5764 struct mem_cgroup *from, 5765 struct mem_cgroup *to) 5766 { 5767 struct folio *folio = page_folio(page); 5768 struct lruvec *from_vec, *to_vec; 5769 struct pglist_data *pgdat; 5770 unsigned int nr_pages = compound ? folio_nr_pages(folio) : 1; 5771 int nid, ret; 5772 5773 VM_BUG_ON(from == to); 5774 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 5775 VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); 5776 VM_BUG_ON(compound && !folio_test_large(folio)); 5777 5778 ret = -EINVAL; 5779 if (folio_memcg(folio) != from) 5780 goto out; 5781 5782 pgdat = folio_pgdat(folio); 5783 from_vec = mem_cgroup_lruvec(from, pgdat); 5784 to_vec = mem_cgroup_lruvec(to, pgdat); 5785 5786 folio_memcg_lock(folio); 5787 5788 if (folio_test_anon(folio)) { 5789 if (folio_mapped(folio)) { 5790 __mod_lruvec_state(from_vec, NR_ANON_MAPPED, -nr_pages); 5791 __mod_lruvec_state(to_vec, NR_ANON_MAPPED, nr_pages); 5792 if (folio_test_pmd_mappable(folio)) { 5793 __mod_lruvec_state(from_vec, NR_ANON_THPS, 5794 -nr_pages); 5795 __mod_lruvec_state(to_vec, NR_ANON_THPS, 5796 nr_pages); 5797 } 5798 } 5799 } else { 5800 __mod_lruvec_state(from_vec, NR_FILE_PAGES, -nr_pages); 5801 __mod_lruvec_state(to_vec, NR_FILE_PAGES, nr_pages); 5802 5803 if (folio_test_swapbacked(folio)) { 5804 __mod_lruvec_state(from_vec, NR_SHMEM, -nr_pages); 5805 __mod_lruvec_state(to_vec, NR_SHMEM, nr_pages); 5806 } 5807 5808 if (folio_mapped(folio)) { 5809 __mod_lruvec_state(from_vec, NR_FILE_MAPPED, -nr_pages); 5810 __mod_lruvec_state(to_vec, NR_FILE_MAPPED, nr_pages); 5811 } 5812 5813 if (folio_test_dirty(folio)) { 5814 struct address_space *mapping = folio_mapping(folio); 5815 5816 if (mapping_can_writeback(mapping)) { 5817 __mod_lruvec_state(from_vec, NR_FILE_DIRTY, 5818 -nr_pages); 5819 __mod_lruvec_state(to_vec, NR_FILE_DIRTY, 5820 nr_pages); 5821 } 5822 } 5823 } 5824 5825 #ifdef CONFIG_SWAP 5826 if (folio_test_swapcache(folio)) { 5827 __mod_lruvec_state(from_vec, NR_SWAPCACHE, -nr_pages); 5828 __mod_lruvec_state(to_vec, NR_SWAPCACHE, nr_pages); 5829 } 5830 #endif 5831 if (folio_test_writeback(folio)) { 5832 __mod_lruvec_state(from_vec, NR_WRITEBACK, -nr_pages); 5833 __mod_lruvec_state(to_vec, NR_WRITEBACK, nr_pages); 5834 } 5835 5836 /* 5837 * All state has been migrated, let's switch to the new memcg. 5838 * 5839 * It is safe to change page's memcg here because the page 5840 * is referenced, charged, isolated, and locked: we can't race 5841 * with (un)charging, migration, LRU putback, or anything else 5842 * that would rely on a stable page's memory cgroup. 5843 * 5844 * Note that folio_memcg_lock is a memcg lock, not a page lock, 5845 * to save space. As soon as we switch page's memory cgroup to a 5846 * new memcg that isn't locked, the above state can change 5847 * concurrently again. Make sure we're truly done with it. 5848 */ 5849 smp_mb(); 5850 5851 css_get(&to->css); 5852 css_put(&from->css); 5853 5854 folio->memcg_data = (unsigned long)to; 5855 5856 __folio_memcg_unlock(from); 5857 5858 ret = 0; 5859 nid = folio_nid(folio); 5860 5861 local_irq_disable(); 5862 mem_cgroup_charge_statistics(to, nr_pages); 5863 memcg_check_events(to, nid); 5864 mem_cgroup_charge_statistics(from, -nr_pages); 5865 memcg_check_events(from, nid); 5866 local_irq_enable(); 5867 out: 5868 return ret; 5869 } 5870 5871 /** 5872 * get_mctgt_type - get target type of moving charge 5873 * @vma: the vma the pte to be checked belongs 5874 * @addr: the address corresponding to the pte to be checked 5875 * @ptent: the pte to be checked 5876 * @target: the pointer the target page or swap ent will be stored(can be NULL) 5877 * 5878 * Context: Called with pte lock held. 5879 * Return: 5880 * * MC_TARGET_NONE - If the pte is not a target for move charge. 5881 * * MC_TARGET_PAGE - If the page corresponding to this pte is a target for 5882 * move charge. If @target is not NULL, the page is stored in target->page 5883 * with extra refcnt taken (Caller should release it). 5884 * * MC_TARGET_SWAP - If the swap entry corresponding to this pte is a 5885 * target for charge migration. If @target is not NULL, the entry is 5886 * stored in target->ent. 5887 * * MC_TARGET_DEVICE - Like MC_TARGET_PAGE but page is device memory and 5888 * thus not on the lru. For now such page is charged like a regular page 5889 * would be as it is just special memory taking the place of a regular page. 5890 * See Documentations/vm/hmm.txt and include/linux/hmm.h 5891 */ 5892 static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma, 5893 unsigned long addr, pte_t ptent, union mc_target *target) 5894 { 5895 struct page *page = NULL; 5896 enum mc_target_type ret = MC_TARGET_NONE; 5897 swp_entry_t ent = { .val = 0 }; 5898 5899 if (pte_present(ptent)) 5900 page = mc_handle_present_pte(vma, addr, ptent); 5901 else if (pte_none_mostly(ptent)) 5902 /* 5903 * PTE markers should be treated as a none pte here, separated 5904 * from other swap handling below. 5905 */ 5906 page = mc_handle_file_pte(vma, addr, ptent); 5907 else if (is_swap_pte(ptent)) 5908 page = mc_handle_swap_pte(vma, ptent, &ent); 5909 5910 if (target && page) { 5911 if (!trylock_page(page)) { 5912 put_page(page); 5913 return ret; 5914 } 5915 /* 5916 * page_mapped() must be stable during the move. This 5917 * pte is locked, so if it's present, the page cannot 5918 * become unmapped. If it isn't, we have only partial 5919 * control over the mapped state: the page lock will 5920 * prevent new faults against pagecache and swapcache, 5921 * so an unmapped page cannot become mapped. However, 5922 * if the page is already mapped elsewhere, it can 5923 * unmap, and there is nothing we can do about it. 5924 * Alas, skip moving the page in this case. 5925 */ 5926 if (!pte_present(ptent) && page_mapped(page)) { 5927 unlock_page(page); 5928 put_page(page); 5929 return ret; 5930 } 5931 } 5932 5933 if (!page && !ent.val) 5934 return ret; 5935 if (page) { 5936 /* 5937 * Do only loose check w/o serialization. 5938 * mem_cgroup_move_account() checks the page is valid or 5939 * not under LRU exclusion. 5940 */ 5941 if (page_memcg(page) == mc.from) { 5942 ret = MC_TARGET_PAGE; 5943 if (is_device_private_page(page) || 5944 is_device_coherent_page(page)) 5945 ret = MC_TARGET_DEVICE; 5946 if (target) 5947 target->page = page; 5948 } 5949 if (!ret || !target) { 5950 if (target) 5951 unlock_page(page); 5952 put_page(page); 5953 } 5954 } 5955 /* 5956 * There is a swap entry and a page doesn't exist or isn't charged. 5957 * But we cannot move a tail-page in a THP. 5958 */ 5959 if (ent.val && !ret && (!page || !PageTransCompound(page)) && 5960 mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) { 5961 ret = MC_TARGET_SWAP; 5962 if (target) 5963 target->ent = ent; 5964 } 5965 return ret; 5966 } 5967 5968 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5969 /* 5970 * We don't consider PMD mapped swapping or file mapped pages because THP does 5971 * not support them for now. 5972 * Caller should make sure that pmd_trans_huge(pmd) is true. 5973 */ 5974 static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma, 5975 unsigned long addr, pmd_t pmd, union mc_target *target) 5976 { 5977 struct page *page = NULL; 5978 enum mc_target_type ret = MC_TARGET_NONE; 5979 5980 if (unlikely(is_swap_pmd(pmd))) { 5981 VM_BUG_ON(thp_migration_supported() && 5982 !is_pmd_migration_entry(pmd)); 5983 return ret; 5984 } 5985 page = pmd_page(pmd); 5986 VM_BUG_ON_PAGE(!page || !PageHead(page), page); 5987 if (!(mc.flags & MOVE_ANON)) 5988 return ret; 5989 if (page_memcg(page) == mc.from) { 5990 ret = MC_TARGET_PAGE; 5991 if (target) { 5992 get_page(page); 5993 if (!trylock_page(page)) { 5994 put_page(page); 5995 return MC_TARGET_NONE; 5996 } 5997 target->page = page; 5998 } 5999 } 6000 return ret; 6001 } 6002 #else 6003 static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma, 6004 unsigned long addr, pmd_t pmd, union mc_target *target) 6005 { 6006 return MC_TARGET_NONE; 6007 } 6008 #endif 6009 6010 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd, 6011 unsigned long addr, unsigned long end, 6012 struct mm_walk *walk) 6013 { 6014 struct vm_area_struct *vma = walk->vma; 6015 pte_t *pte; 6016 spinlock_t *ptl; 6017 6018 ptl = pmd_trans_huge_lock(pmd, vma); 6019 if (ptl) { 6020 /* 6021 * Note their can not be MC_TARGET_DEVICE for now as we do not 6022 * support transparent huge page with MEMORY_DEVICE_PRIVATE but 6023 * this might change. 6024 */ 6025 if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE) 6026 mc.precharge += HPAGE_PMD_NR; 6027 spin_unlock(ptl); 6028 return 0; 6029 } 6030 6031 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 6032 if (!pte) 6033 return 0; 6034 for (; addr != end; pte++, addr += PAGE_SIZE) 6035 if (get_mctgt_type(vma, addr, ptep_get(pte), NULL)) 6036 mc.precharge++; /* increment precharge temporarily */ 6037 pte_unmap_unlock(pte - 1, ptl); 6038 cond_resched(); 6039 6040 return 0; 6041 } 6042 6043 static const struct mm_walk_ops precharge_walk_ops = { 6044 .pmd_entry = mem_cgroup_count_precharge_pte_range, 6045 .walk_lock = PGWALK_RDLOCK, 6046 }; 6047 6048 static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm) 6049 { 6050 unsigned long precharge; 6051 6052 mmap_read_lock(mm); 6053 walk_page_range(mm, 0, ULONG_MAX, &precharge_walk_ops, NULL); 6054 mmap_read_unlock(mm); 6055 6056 precharge = mc.precharge; 6057 mc.precharge = 0; 6058 6059 return precharge; 6060 } 6061 6062 static int mem_cgroup_precharge_mc(struct mm_struct *mm) 6063 { 6064 unsigned long precharge = mem_cgroup_count_precharge(mm); 6065 6066 VM_BUG_ON(mc.moving_task); 6067 mc.moving_task = current; 6068 return mem_cgroup_do_precharge(precharge); 6069 } 6070 6071 /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */ 6072 static void __mem_cgroup_clear_mc(void) 6073 { 6074 struct mem_cgroup *from = mc.from; 6075 struct mem_cgroup *to = mc.to; 6076 6077 /* we must uncharge all the leftover precharges from mc.to */ 6078 if (mc.precharge) { 6079 cancel_charge(mc.to, mc.precharge); 6080 mc.precharge = 0; 6081 } 6082 /* 6083 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so 6084 * we must uncharge here. 6085 */ 6086 if (mc.moved_charge) { 6087 cancel_charge(mc.from, mc.moved_charge); 6088 mc.moved_charge = 0; 6089 } 6090 /* we must fixup refcnts and charges */ 6091 if (mc.moved_swap) { 6092 /* uncharge swap account from the old cgroup */ 6093 if (!mem_cgroup_is_root(mc.from)) 6094 page_counter_uncharge(&mc.from->memsw, mc.moved_swap); 6095 6096 mem_cgroup_id_put_many(mc.from, mc.moved_swap); 6097 6098 /* 6099 * we charged both to->memory and to->memsw, so we 6100 * should uncharge to->memory. 6101 */ 6102 if (!mem_cgroup_is_root(mc.to)) 6103 page_counter_uncharge(&mc.to->memory, mc.moved_swap); 6104 6105 mc.moved_swap = 0; 6106 } 6107 memcg_oom_recover(from); 6108 memcg_oom_recover(to); 6109 wake_up_all(&mc.waitq); 6110 } 6111 6112 static void mem_cgroup_clear_mc(void) 6113 { 6114 struct mm_struct *mm = mc.mm; 6115 6116 /* 6117 * we must clear moving_task before waking up waiters at the end of 6118 * task migration. 6119 */ 6120 mc.moving_task = NULL; 6121 __mem_cgroup_clear_mc(); 6122 spin_lock(&mc.lock); 6123 mc.from = NULL; 6124 mc.to = NULL; 6125 mc.mm = NULL; 6126 spin_unlock(&mc.lock); 6127 6128 mmput(mm); 6129 } 6130 6131 static int mem_cgroup_can_attach(struct cgroup_taskset *tset) 6132 { 6133 struct cgroup_subsys_state *css; 6134 struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */ 6135 struct mem_cgroup *from; 6136 struct task_struct *leader, *p; 6137 struct mm_struct *mm; 6138 unsigned long move_flags; 6139 int ret = 0; 6140 6141 /* charge immigration isn't supported on the default hierarchy */ 6142 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 6143 return 0; 6144 6145 /* 6146 * Multi-process migrations only happen on the default hierarchy 6147 * where charge immigration is not used. Perform charge 6148 * immigration if @tset contains a leader and whine if there are 6149 * multiple. 6150 */ 6151 p = NULL; 6152 cgroup_taskset_for_each_leader(leader, css, tset) { 6153 WARN_ON_ONCE(p); 6154 p = leader; 6155 memcg = mem_cgroup_from_css(css); 6156 } 6157 if (!p) 6158 return 0; 6159 6160 /* 6161 * We are now committed to this value whatever it is. Changes in this 6162 * tunable will only affect upcoming migrations, not the current one. 6163 * So we need to save it, and keep it going. 6164 */ 6165 move_flags = READ_ONCE(memcg->move_charge_at_immigrate); 6166 if (!move_flags) 6167 return 0; 6168 6169 from = mem_cgroup_from_task(p); 6170 6171 VM_BUG_ON(from == memcg); 6172 6173 mm = get_task_mm(p); 6174 if (!mm) 6175 return 0; 6176 /* We move charges only when we move a owner of the mm */ 6177 if (mm->owner == p) { 6178 VM_BUG_ON(mc.from); 6179 VM_BUG_ON(mc.to); 6180 VM_BUG_ON(mc.precharge); 6181 VM_BUG_ON(mc.moved_charge); 6182 VM_BUG_ON(mc.moved_swap); 6183 6184 spin_lock(&mc.lock); 6185 mc.mm = mm; 6186 mc.from = from; 6187 mc.to = memcg; 6188 mc.flags = move_flags; 6189 spin_unlock(&mc.lock); 6190 /* We set mc.moving_task later */ 6191 6192 ret = mem_cgroup_precharge_mc(mm); 6193 if (ret) 6194 mem_cgroup_clear_mc(); 6195 } else { 6196 mmput(mm); 6197 } 6198 return ret; 6199 } 6200 6201 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset) 6202 { 6203 if (mc.to) 6204 mem_cgroup_clear_mc(); 6205 } 6206 6207 static int mem_cgroup_move_charge_pte_range(pmd_t *pmd, 6208 unsigned long addr, unsigned long end, 6209 struct mm_walk *walk) 6210 { 6211 int ret = 0; 6212 struct vm_area_struct *vma = walk->vma; 6213 pte_t *pte; 6214 spinlock_t *ptl; 6215 enum mc_target_type target_type; 6216 union mc_target target; 6217 struct page *page; 6218 6219 ptl = pmd_trans_huge_lock(pmd, vma); 6220 if (ptl) { 6221 if (mc.precharge < HPAGE_PMD_NR) { 6222 spin_unlock(ptl); 6223 return 0; 6224 } 6225 target_type = get_mctgt_type_thp(vma, addr, *pmd, &target); 6226 if (target_type == MC_TARGET_PAGE) { 6227 page = target.page; 6228 if (isolate_lru_page(page)) { 6229 if (!mem_cgroup_move_account(page, true, 6230 mc.from, mc.to)) { 6231 mc.precharge -= HPAGE_PMD_NR; 6232 mc.moved_charge += HPAGE_PMD_NR; 6233 } 6234 putback_lru_page(page); 6235 } 6236 unlock_page(page); 6237 put_page(page); 6238 } else if (target_type == MC_TARGET_DEVICE) { 6239 page = target.page; 6240 if (!mem_cgroup_move_account(page, true, 6241 mc.from, mc.to)) { 6242 mc.precharge -= HPAGE_PMD_NR; 6243 mc.moved_charge += HPAGE_PMD_NR; 6244 } 6245 unlock_page(page); 6246 put_page(page); 6247 } 6248 spin_unlock(ptl); 6249 return 0; 6250 } 6251 6252 retry: 6253 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 6254 if (!pte) 6255 return 0; 6256 for (; addr != end; addr += PAGE_SIZE) { 6257 pte_t ptent = ptep_get(pte++); 6258 bool device = false; 6259 swp_entry_t ent; 6260 6261 if (!mc.precharge) 6262 break; 6263 6264 switch (get_mctgt_type(vma, addr, ptent, &target)) { 6265 case MC_TARGET_DEVICE: 6266 device = true; 6267 fallthrough; 6268 case MC_TARGET_PAGE: 6269 page = target.page; 6270 /* 6271 * We can have a part of the split pmd here. Moving it 6272 * can be done but it would be too convoluted so simply 6273 * ignore such a partial THP and keep it in original 6274 * memcg. There should be somebody mapping the head. 6275 */ 6276 if (PageTransCompound(page)) 6277 goto put; 6278 if (!device && !isolate_lru_page(page)) 6279 goto put; 6280 if (!mem_cgroup_move_account(page, false, 6281 mc.from, mc.to)) { 6282 mc.precharge--; 6283 /* we uncharge from mc.from later. */ 6284 mc.moved_charge++; 6285 } 6286 if (!device) 6287 putback_lru_page(page); 6288 put: /* get_mctgt_type() gets & locks the page */ 6289 unlock_page(page); 6290 put_page(page); 6291 break; 6292 case MC_TARGET_SWAP: 6293 ent = target.ent; 6294 if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) { 6295 mc.precharge--; 6296 mem_cgroup_id_get_many(mc.to, 1); 6297 /* we fixup other refcnts and charges later. */ 6298 mc.moved_swap++; 6299 } 6300 break; 6301 default: 6302 break; 6303 } 6304 } 6305 pte_unmap_unlock(pte - 1, ptl); 6306 cond_resched(); 6307 6308 if (addr != end) { 6309 /* 6310 * We have consumed all precharges we got in can_attach(). 6311 * We try charge one by one, but don't do any additional 6312 * charges to mc.to if we have failed in charge once in attach() 6313 * phase. 6314 */ 6315 ret = mem_cgroup_do_precharge(1); 6316 if (!ret) 6317 goto retry; 6318 } 6319 6320 return ret; 6321 } 6322 6323 static const struct mm_walk_ops charge_walk_ops = { 6324 .pmd_entry = mem_cgroup_move_charge_pte_range, 6325 .walk_lock = PGWALK_RDLOCK, 6326 }; 6327 6328 static void mem_cgroup_move_charge(void) 6329 { 6330 lru_add_drain_all(); 6331 /* 6332 * Signal folio_memcg_lock() to take the memcg's move_lock 6333 * while we're moving its pages to another memcg. Then wait 6334 * for already started RCU-only updates to finish. 6335 */ 6336 atomic_inc(&mc.from->moving_account); 6337 synchronize_rcu(); 6338 retry: 6339 if (unlikely(!mmap_read_trylock(mc.mm))) { 6340 /* 6341 * Someone who are holding the mmap_lock might be waiting in 6342 * waitq. So we cancel all extra charges, wake up all waiters, 6343 * and retry. Because we cancel precharges, we might not be able 6344 * to move enough charges, but moving charge is a best-effort 6345 * feature anyway, so it wouldn't be a big problem. 6346 */ 6347 __mem_cgroup_clear_mc(); 6348 cond_resched(); 6349 goto retry; 6350 } 6351 /* 6352 * When we have consumed all precharges and failed in doing 6353 * additional charge, the page walk just aborts. 6354 */ 6355 walk_page_range(mc.mm, 0, ULONG_MAX, &charge_walk_ops, NULL); 6356 mmap_read_unlock(mc.mm); 6357 atomic_dec(&mc.from->moving_account); 6358 } 6359 6360 static void mem_cgroup_move_task(void) 6361 { 6362 if (mc.to) { 6363 mem_cgroup_move_charge(); 6364 mem_cgroup_clear_mc(); 6365 } 6366 } 6367 #else /* !CONFIG_MMU */ 6368 static int mem_cgroup_can_attach(struct cgroup_taskset *tset) 6369 { 6370 return 0; 6371 } 6372 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset) 6373 { 6374 } 6375 static void mem_cgroup_move_task(void) 6376 { 6377 } 6378 #endif 6379 6380 #ifdef CONFIG_LRU_GEN 6381 static void mem_cgroup_attach(struct cgroup_taskset *tset) 6382 { 6383 struct task_struct *task; 6384 struct cgroup_subsys_state *css; 6385 6386 /* find the first leader if there is any */ 6387 cgroup_taskset_for_each_leader(task, css, tset) 6388 break; 6389 6390 if (!task) 6391 return; 6392 6393 task_lock(task); 6394 if (task->mm && READ_ONCE(task->mm->owner) == task) 6395 lru_gen_migrate_mm(task->mm); 6396 task_unlock(task); 6397 } 6398 #else 6399 static void mem_cgroup_attach(struct cgroup_taskset *tset) 6400 { 6401 } 6402 #endif /* CONFIG_LRU_GEN */ 6403 6404 static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value) 6405 { 6406 if (value == PAGE_COUNTER_MAX) 6407 seq_puts(m, "max\n"); 6408 else 6409 seq_printf(m, "%llu\n", (u64)value * PAGE_SIZE); 6410 6411 return 0; 6412 } 6413 6414 static u64 memory_current_read(struct cgroup_subsys_state *css, 6415 struct cftype *cft) 6416 { 6417 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 6418 6419 return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE; 6420 } 6421 6422 static u64 memory_peak_read(struct cgroup_subsys_state *css, 6423 struct cftype *cft) 6424 { 6425 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 6426 6427 return (u64)memcg->memory.watermark * PAGE_SIZE; 6428 } 6429 6430 static int memory_min_show(struct seq_file *m, void *v) 6431 { 6432 return seq_puts_memcg_tunable(m, 6433 READ_ONCE(mem_cgroup_from_seq(m)->memory.min)); 6434 } 6435 6436 static ssize_t memory_min_write(struct kernfs_open_file *of, 6437 char *buf, size_t nbytes, loff_t off) 6438 { 6439 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 6440 unsigned long min; 6441 int err; 6442 6443 buf = strstrip(buf); 6444 err = page_counter_memparse(buf, "max", &min); 6445 if (err) 6446 return err; 6447 6448 page_counter_set_min(&memcg->memory, min); 6449 6450 return nbytes; 6451 } 6452 6453 static int memory_low_show(struct seq_file *m, void *v) 6454 { 6455 return seq_puts_memcg_tunable(m, 6456 READ_ONCE(mem_cgroup_from_seq(m)->memory.low)); 6457 } 6458 6459 static ssize_t memory_low_write(struct kernfs_open_file *of, 6460 char *buf, size_t nbytes, loff_t off) 6461 { 6462 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 6463 unsigned long low; 6464 int err; 6465 6466 buf = strstrip(buf); 6467 err = page_counter_memparse(buf, "max", &low); 6468 if (err) 6469 return err; 6470 6471 page_counter_set_low(&memcg->memory, low); 6472 6473 return nbytes; 6474 } 6475 6476 static int memory_high_show(struct seq_file *m, void *v) 6477 { 6478 return seq_puts_memcg_tunable(m, 6479 READ_ONCE(mem_cgroup_from_seq(m)->memory.high)); 6480 } 6481 6482 static ssize_t memory_high_write(struct kernfs_open_file *of, 6483 char *buf, size_t nbytes, loff_t off) 6484 { 6485 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 6486 unsigned int nr_retries = MAX_RECLAIM_RETRIES; 6487 bool drained = false; 6488 unsigned long high; 6489 int err; 6490 6491 buf = strstrip(buf); 6492 err = page_counter_memparse(buf, "max", &high); 6493 if (err) 6494 return err; 6495 6496 page_counter_set_high(&memcg->memory, high); 6497 6498 for (;;) { 6499 unsigned long nr_pages = page_counter_read(&memcg->memory); 6500 unsigned long reclaimed; 6501 6502 if (nr_pages <= high) 6503 break; 6504 6505 if (signal_pending(current)) 6506 break; 6507 6508 if (!drained) { 6509 drain_all_stock(memcg); 6510 drained = true; 6511 continue; 6512 } 6513 6514 reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high, 6515 GFP_KERNEL, MEMCG_RECLAIM_MAY_SWAP); 6516 6517 if (!reclaimed && !nr_retries--) 6518 break; 6519 } 6520 6521 memcg_wb_domain_size_changed(memcg); 6522 return nbytes; 6523 } 6524 6525 static int memory_max_show(struct seq_file *m, void *v) 6526 { 6527 return seq_puts_memcg_tunable(m, 6528 READ_ONCE(mem_cgroup_from_seq(m)->memory.max)); 6529 } 6530 6531 static ssize_t memory_max_write(struct kernfs_open_file *of, 6532 char *buf, size_t nbytes, loff_t off) 6533 { 6534 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 6535 unsigned int nr_reclaims = MAX_RECLAIM_RETRIES; 6536 bool drained = false; 6537 unsigned long max; 6538 int err; 6539 6540 buf = strstrip(buf); 6541 err = page_counter_memparse(buf, "max", &max); 6542 if (err) 6543 return err; 6544 6545 xchg(&memcg->memory.max, max); 6546 6547 for (;;) { 6548 unsigned long nr_pages = page_counter_read(&memcg->memory); 6549 6550 if (nr_pages <= max) 6551 break; 6552 6553 if (signal_pending(current)) 6554 break; 6555 6556 if (!drained) { 6557 drain_all_stock(memcg); 6558 drained = true; 6559 continue; 6560 } 6561 6562 if (nr_reclaims) { 6563 if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max, 6564 GFP_KERNEL, MEMCG_RECLAIM_MAY_SWAP)) 6565 nr_reclaims--; 6566 continue; 6567 } 6568 6569 memcg_memory_event(memcg, MEMCG_OOM); 6570 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0)) 6571 break; 6572 } 6573 6574 memcg_wb_domain_size_changed(memcg); 6575 return nbytes; 6576 } 6577 6578 static void __memory_events_show(struct seq_file *m, atomic_long_t *events) 6579 { 6580 seq_printf(m, "low %lu\n", atomic_long_read(&events[MEMCG_LOW])); 6581 seq_printf(m, "high %lu\n", atomic_long_read(&events[MEMCG_HIGH])); 6582 seq_printf(m, "max %lu\n", atomic_long_read(&events[MEMCG_MAX])); 6583 seq_printf(m, "oom %lu\n", atomic_long_read(&events[MEMCG_OOM])); 6584 seq_printf(m, "oom_kill %lu\n", 6585 atomic_long_read(&events[MEMCG_OOM_KILL])); 6586 seq_printf(m, "oom_group_kill %lu\n", 6587 atomic_long_read(&events[MEMCG_OOM_GROUP_KILL])); 6588 } 6589 6590 static int memory_events_show(struct seq_file *m, void *v) 6591 { 6592 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 6593 6594 __memory_events_show(m, memcg->memory_events); 6595 return 0; 6596 } 6597 6598 static int memory_events_local_show(struct seq_file *m, void *v) 6599 { 6600 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 6601 6602 __memory_events_show(m, memcg->memory_events_local); 6603 return 0; 6604 } 6605 6606 static int memory_stat_show(struct seq_file *m, void *v) 6607 { 6608 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 6609 char *buf = kmalloc(PAGE_SIZE, GFP_KERNEL); 6610 struct seq_buf s; 6611 6612 if (!buf) 6613 return -ENOMEM; 6614 seq_buf_init(&s, buf, PAGE_SIZE); 6615 memory_stat_format(memcg, &s); 6616 seq_puts(m, buf); 6617 kfree(buf); 6618 return 0; 6619 } 6620 6621 #ifdef CONFIG_NUMA 6622 static inline unsigned long lruvec_page_state_output(struct lruvec *lruvec, 6623 int item) 6624 { 6625 return lruvec_page_state(lruvec, item) * memcg_page_state_unit(item); 6626 } 6627 6628 static int memory_numa_stat_show(struct seq_file *m, void *v) 6629 { 6630 int i; 6631 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 6632 6633 mem_cgroup_flush_stats(); 6634 6635 for (i = 0; i < ARRAY_SIZE(memory_stats); i++) { 6636 int nid; 6637 6638 if (memory_stats[i].idx >= NR_VM_NODE_STAT_ITEMS) 6639 continue; 6640 6641 seq_printf(m, "%s", memory_stats[i].name); 6642 for_each_node_state(nid, N_MEMORY) { 6643 u64 size; 6644 struct lruvec *lruvec; 6645 6646 lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid)); 6647 size = lruvec_page_state_output(lruvec, 6648 memory_stats[i].idx); 6649 seq_printf(m, " N%d=%llu", nid, size); 6650 } 6651 seq_putc(m, '\n'); 6652 } 6653 6654 return 0; 6655 } 6656 #endif 6657 6658 static int memory_oom_group_show(struct seq_file *m, void *v) 6659 { 6660 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 6661 6662 seq_printf(m, "%d\n", READ_ONCE(memcg->oom_group)); 6663 6664 return 0; 6665 } 6666 6667 static ssize_t memory_oom_group_write(struct kernfs_open_file *of, 6668 char *buf, size_t nbytes, loff_t off) 6669 { 6670 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 6671 int ret, oom_group; 6672 6673 buf = strstrip(buf); 6674 if (!buf) 6675 return -EINVAL; 6676 6677 ret = kstrtoint(buf, 0, &oom_group); 6678 if (ret) 6679 return ret; 6680 6681 if (oom_group != 0 && oom_group != 1) 6682 return -EINVAL; 6683 6684 WRITE_ONCE(memcg->oom_group, oom_group); 6685 6686 return nbytes; 6687 } 6688 6689 static ssize_t memory_reclaim(struct kernfs_open_file *of, char *buf, 6690 size_t nbytes, loff_t off) 6691 { 6692 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 6693 unsigned int nr_retries = MAX_RECLAIM_RETRIES; 6694 unsigned long nr_to_reclaim, nr_reclaimed = 0; 6695 unsigned int reclaim_options; 6696 int err; 6697 6698 buf = strstrip(buf); 6699 err = page_counter_memparse(buf, "", &nr_to_reclaim); 6700 if (err) 6701 return err; 6702 6703 reclaim_options = MEMCG_RECLAIM_MAY_SWAP | MEMCG_RECLAIM_PROACTIVE; 6704 while (nr_reclaimed < nr_to_reclaim) { 6705 unsigned long reclaimed; 6706 6707 if (signal_pending(current)) 6708 return -EINTR; 6709 6710 /* 6711 * This is the final attempt, drain percpu lru caches in the 6712 * hope of introducing more evictable pages for 6713 * try_to_free_mem_cgroup_pages(). 6714 */ 6715 if (!nr_retries) 6716 lru_add_drain_all(); 6717 6718 reclaimed = try_to_free_mem_cgroup_pages(memcg, 6719 min(nr_to_reclaim - nr_reclaimed, SWAP_CLUSTER_MAX), 6720 GFP_KERNEL, reclaim_options); 6721 6722 if (!reclaimed && !nr_retries--) 6723 return -EAGAIN; 6724 6725 nr_reclaimed += reclaimed; 6726 } 6727 6728 return nbytes; 6729 } 6730 6731 static struct cftype memory_files[] = { 6732 { 6733 .name = "current", 6734 .flags = CFTYPE_NOT_ON_ROOT, 6735 .read_u64 = memory_current_read, 6736 }, 6737 { 6738 .name = "peak", 6739 .flags = CFTYPE_NOT_ON_ROOT, 6740 .read_u64 = memory_peak_read, 6741 }, 6742 { 6743 .name = "min", 6744 .flags = CFTYPE_NOT_ON_ROOT, 6745 .seq_show = memory_min_show, 6746 .write = memory_min_write, 6747 }, 6748 { 6749 .name = "low", 6750 .flags = CFTYPE_NOT_ON_ROOT, 6751 .seq_show = memory_low_show, 6752 .write = memory_low_write, 6753 }, 6754 { 6755 .name = "high", 6756 .flags = CFTYPE_NOT_ON_ROOT, 6757 .seq_show = memory_high_show, 6758 .write = memory_high_write, 6759 }, 6760 { 6761 .name = "max", 6762 .flags = CFTYPE_NOT_ON_ROOT, 6763 .seq_show = memory_max_show, 6764 .write = memory_max_write, 6765 }, 6766 { 6767 .name = "events", 6768 .flags = CFTYPE_NOT_ON_ROOT, 6769 .file_offset = offsetof(struct mem_cgroup, events_file), 6770 .seq_show = memory_events_show, 6771 }, 6772 { 6773 .name = "events.local", 6774 .flags = CFTYPE_NOT_ON_ROOT, 6775 .file_offset = offsetof(struct mem_cgroup, events_local_file), 6776 .seq_show = memory_events_local_show, 6777 }, 6778 { 6779 .name = "stat", 6780 .seq_show = memory_stat_show, 6781 }, 6782 #ifdef CONFIG_NUMA 6783 { 6784 .name = "numa_stat", 6785 .seq_show = memory_numa_stat_show, 6786 }, 6787 #endif 6788 { 6789 .name = "oom.group", 6790 .flags = CFTYPE_NOT_ON_ROOT | CFTYPE_NS_DELEGATABLE, 6791 .seq_show = memory_oom_group_show, 6792 .write = memory_oom_group_write, 6793 }, 6794 { 6795 .name = "reclaim", 6796 .flags = CFTYPE_NS_DELEGATABLE, 6797 .write = memory_reclaim, 6798 }, 6799 { } /* terminate */ 6800 }; 6801 6802 struct cgroup_subsys memory_cgrp_subsys = { 6803 .css_alloc = mem_cgroup_css_alloc, 6804 .css_online = mem_cgroup_css_online, 6805 .css_offline = mem_cgroup_css_offline, 6806 .css_released = mem_cgroup_css_released, 6807 .css_free = mem_cgroup_css_free, 6808 .css_reset = mem_cgroup_css_reset, 6809 .css_rstat_flush = mem_cgroup_css_rstat_flush, 6810 .can_attach = mem_cgroup_can_attach, 6811 .attach = mem_cgroup_attach, 6812 .cancel_attach = mem_cgroup_cancel_attach, 6813 .post_attach = mem_cgroup_move_task, 6814 .dfl_cftypes = memory_files, 6815 .legacy_cftypes = mem_cgroup_legacy_files, 6816 .early_init = 0, 6817 }; 6818 6819 /* 6820 * This function calculates an individual cgroup's effective 6821 * protection which is derived from its own memory.min/low, its 6822 * parent's and siblings' settings, as well as the actual memory 6823 * distribution in the tree. 6824 * 6825 * The following rules apply to the effective protection values: 6826 * 6827 * 1. At the first level of reclaim, effective protection is equal to 6828 * the declared protection in memory.min and memory.low. 6829 * 6830 * 2. To enable safe delegation of the protection configuration, at 6831 * subsequent levels the effective protection is capped to the 6832 * parent's effective protection. 6833 * 6834 * 3. To make complex and dynamic subtrees easier to configure, the 6835 * user is allowed to overcommit the declared protection at a given 6836 * level. If that is the case, the parent's effective protection is 6837 * distributed to the children in proportion to how much protection 6838 * they have declared and how much of it they are utilizing. 6839 * 6840 * This makes distribution proportional, but also work-conserving: 6841 * if one cgroup claims much more protection than it uses memory, 6842 * the unused remainder is available to its siblings. 6843 * 6844 * 4. Conversely, when the declared protection is undercommitted at a 6845 * given level, the distribution of the larger parental protection 6846 * budget is NOT proportional. A cgroup's protection from a sibling 6847 * is capped to its own memory.min/low setting. 6848 * 6849 * 5. However, to allow protecting recursive subtrees from each other 6850 * without having to declare each individual cgroup's fixed share 6851 * of the ancestor's claim to protection, any unutilized - 6852 * "floating" - protection from up the tree is distributed in 6853 * proportion to each cgroup's *usage*. This makes the protection 6854 * neutral wrt sibling cgroups and lets them compete freely over 6855 * the shared parental protection budget, but it protects the 6856 * subtree as a whole from neighboring subtrees. 6857 * 6858 * Note that 4. and 5. are not in conflict: 4. is about protecting 6859 * against immediate siblings whereas 5. is about protecting against 6860 * neighboring subtrees. 6861 */ 6862 static unsigned long effective_protection(unsigned long usage, 6863 unsigned long parent_usage, 6864 unsigned long setting, 6865 unsigned long parent_effective, 6866 unsigned long siblings_protected) 6867 { 6868 unsigned long protected; 6869 unsigned long ep; 6870 6871 protected = min(usage, setting); 6872 /* 6873 * If all cgroups at this level combined claim and use more 6874 * protection than what the parent affords them, distribute 6875 * shares in proportion to utilization. 6876 * 6877 * We are using actual utilization rather than the statically 6878 * claimed protection in order to be work-conserving: claimed 6879 * but unused protection is available to siblings that would 6880 * otherwise get a smaller chunk than what they claimed. 6881 */ 6882 if (siblings_protected > parent_effective) 6883 return protected * parent_effective / siblings_protected; 6884 6885 /* 6886 * Ok, utilized protection of all children is within what the 6887 * parent affords them, so we know whatever this child claims 6888 * and utilizes is effectively protected. 6889 * 6890 * If there is unprotected usage beyond this value, reclaim 6891 * will apply pressure in proportion to that amount. 6892 * 6893 * If there is unutilized protection, the cgroup will be fully 6894 * shielded from reclaim, but we do return a smaller value for 6895 * protection than what the group could enjoy in theory. This 6896 * is okay. With the overcommit distribution above, effective 6897 * protection is always dependent on how memory is actually 6898 * consumed among the siblings anyway. 6899 */ 6900 ep = protected; 6901 6902 /* 6903 * If the children aren't claiming (all of) the protection 6904 * afforded to them by the parent, distribute the remainder in 6905 * proportion to the (unprotected) memory of each cgroup. That 6906 * way, cgroups that aren't explicitly prioritized wrt each 6907 * other compete freely over the allowance, but they are 6908 * collectively protected from neighboring trees. 6909 * 6910 * We're using unprotected memory for the weight so that if 6911 * some cgroups DO claim explicit protection, we don't protect 6912 * the same bytes twice. 6913 * 6914 * Check both usage and parent_usage against the respective 6915 * protected values. One should imply the other, but they 6916 * aren't read atomically - make sure the division is sane. 6917 */ 6918 if (!(cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT)) 6919 return ep; 6920 if (parent_effective > siblings_protected && 6921 parent_usage > siblings_protected && 6922 usage > protected) { 6923 unsigned long unclaimed; 6924 6925 unclaimed = parent_effective - siblings_protected; 6926 unclaimed *= usage - protected; 6927 unclaimed /= parent_usage - siblings_protected; 6928 6929 ep += unclaimed; 6930 } 6931 6932 return ep; 6933 } 6934 6935 /** 6936 * mem_cgroup_calculate_protection - check if memory consumption is in the normal range 6937 * @root: the top ancestor of the sub-tree being checked 6938 * @memcg: the memory cgroup to check 6939 * 6940 * WARNING: This function is not stateless! It can only be used as part 6941 * of a top-down tree iteration, not for isolated queries. 6942 */ 6943 void mem_cgroup_calculate_protection(struct mem_cgroup *root, 6944 struct mem_cgroup *memcg) 6945 { 6946 unsigned long usage, parent_usage; 6947 struct mem_cgroup *parent; 6948 6949 if (mem_cgroup_disabled()) 6950 return; 6951 6952 if (!root) 6953 root = root_mem_cgroup; 6954 6955 /* 6956 * Effective values of the reclaim targets are ignored so they 6957 * can be stale. Have a look at mem_cgroup_protection for more 6958 * details. 6959 * TODO: calculation should be more robust so that we do not need 6960 * that special casing. 6961 */ 6962 if (memcg == root) 6963 return; 6964 6965 usage = page_counter_read(&memcg->memory); 6966 if (!usage) 6967 return; 6968 6969 parent = parent_mem_cgroup(memcg); 6970 6971 if (parent == root) { 6972 memcg->memory.emin = READ_ONCE(memcg->memory.min); 6973 memcg->memory.elow = READ_ONCE(memcg->memory.low); 6974 return; 6975 } 6976 6977 parent_usage = page_counter_read(&parent->memory); 6978 6979 WRITE_ONCE(memcg->memory.emin, effective_protection(usage, parent_usage, 6980 READ_ONCE(memcg->memory.min), 6981 READ_ONCE(parent->memory.emin), 6982 atomic_long_read(&parent->memory.children_min_usage))); 6983 6984 WRITE_ONCE(memcg->memory.elow, effective_protection(usage, parent_usage, 6985 READ_ONCE(memcg->memory.low), 6986 READ_ONCE(parent->memory.elow), 6987 atomic_long_read(&parent->memory.children_low_usage))); 6988 } 6989 6990 static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg, 6991 gfp_t gfp) 6992 { 6993 long nr_pages = folio_nr_pages(folio); 6994 int ret; 6995 6996 ret = try_charge(memcg, gfp, nr_pages); 6997 if (ret) 6998 goto out; 6999 7000 css_get(&memcg->css); 7001 commit_charge(folio, memcg); 7002 7003 local_irq_disable(); 7004 mem_cgroup_charge_statistics(memcg, nr_pages); 7005 memcg_check_events(memcg, folio_nid(folio)); 7006 local_irq_enable(); 7007 out: 7008 return ret; 7009 } 7010 7011 int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp) 7012 { 7013 struct mem_cgroup *memcg; 7014 int ret; 7015 7016 memcg = get_mem_cgroup_from_mm(mm); 7017 ret = charge_memcg(folio, memcg, gfp); 7018 css_put(&memcg->css); 7019 7020 return ret; 7021 } 7022 7023 /** 7024 * mem_cgroup_swapin_charge_folio - Charge a newly allocated folio for swapin. 7025 * @folio: folio to charge. 7026 * @mm: mm context of the victim 7027 * @gfp: reclaim mode 7028 * @entry: swap entry for which the folio is allocated 7029 * 7030 * This function charges a folio allocated for swapin. Please call this before 7031 * adding the folio to the swapcache. 7032 * 7033 * Returns 0 on success. Otherwise, an error code is returned. 7034 */ 7035 int mem_cgroup_swapin_charge_folio(struct folio *folio, struct mm_struct *mm, 7036 gfp_t gfp, swp_entry_t entry) 7037 { 7038 struct mem_cgroup *memcg; 7039 unsigned short id; 7040 int ret; 7041 7042 if (mem_cgroup_disabled()) 7043 return 0; 7044 7045 id = lookup_swap_cgroup_id(entry); 7046 rcu_read_lock(); 7047 memcg = mem_cgroup_from_id(id); 7048 if (!memcg || !css_tryget_online(&memcg->css)) 7049 memcg = get_mem_cgroup_from_mm(mm); 7050 rcu_read_unlock(); 7051 7052 ret = charge_memcg(folio, memcg, gfp); 7053 7054 css_put(&memcg->css); 7055 return ret; 7056 } 7057 7058 /* 7059 * mem_cgroup_swapin_uncharge_swap - uncharge swap slot 7060 * @entry: swap entry for which the page is charged 7061 * 7062 * Call this function after successfully adding the charged page to swapcache. 7063 * 7064 * Note: This function assumes the page for which swap slot is being uncharged 7065 * is order 0 page. 7066 */ 7067 void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry) 7068 { 7069 /* 7070 * Cgroup1's unified memory+swap counter has been charged with the 7071 * new swapcache page, finish the transfer by uncharging the swap 7072 * slot. The swap slot would also get uncharged when it dies, but 7073 * it can stick around indefinitely and we'd count the page twice 7074 * the entire time. 7075 * 7076 * Cgroup2 has separate resource counters for memory and swap, 7077 * so this is a non-issue here. Memory and swap charge lifetimes 7078 * correspond 1:1 to page and swap slot lifetimes: we charge the 7079 * page to memory here, and uncharge swap when the slot is freed. 7080 */ 7081 if (!mem_cgroup_disabled() && do_memsw_account()) { 7082 /* 7083 * The swap entry might not get freed for a long time, 7084 * let's not wait for it. The page already received a 7085 * memory+swap charge, drop the swap entry duplicate. 7086 */ 7087 mem_cgroup_uncharge_swap(entry, 1); 7088 } 7089 } 7090 7091 struct uncharge_gather { 7092 struct mem_cgroup *memcg; 7093 unsigned long nr_memory; 7094 unsigned long pgpgout; 7095 unsigned long nr_kmem; 7096 int nid; 7097 }; 7098 7099 static inline void uncharge_gather_clear(struct uncharge_gather *ug) 7100 { 7101 memset(ug, 0, sizeof(*ug)); 7102 } 7103 7104 static void uncharge_batch(const struct uncharge_gather *ug) 7105 { 7106 unsigned long flags; 7107 7108 if (ug->nr_memory) { 7109 page_counter_uncharge(&ug->memcg->memory, ug->nr_memory); 7110 if (do_memsw_account()) 7111 page_counter_uncharge(&ug->memcg->memsw, ug->nr_memory); 7112 if (ug->nr_kmem) 7113 memcg_account_kmem(ug->memcg, -ug->nr_kmem); 7114 memcg_oom_recover(ug->memcg); 7115 } 7116 7117 local_irq_save(flags); 7118 __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout); 7119 __this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_memory); 7120 memcg_check_events(ug->memcg, ug->nid); 7121 local_irq_restore(flags); 7122 7123 /* drop reference from uncharge_folio */ 7124 css_put(&ug->memcg->css); 7125 } 7126 7127 static void uncharge_folio(struct folio *folio, struct uncharge_gather *ug) 7128 { 7129 long nr_pages; 7130 struct mem_cgroup *memcg; 7131 struct obj_cgroup *objcg; 7132 7133 VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); 7134 7135 /* 7136 * Nobody should be changing or seriously looking at 7137 * folio memcg or objcg at this point, we have fully 7138 * exclusive access to the folio. 7139 */ 7140 if (folio_memcg_kmem(folio)) { 7141 objcg = __folio_objcg(folio); 7142 /* 7143 * This get matches the put at the end of the function and 7144 * kmem pages do not hold memcg references anymore. 7145 */ 7146 memcg = get_mem_cgroup_from_objcg(objcg); 7147 } else { 7148 memcg = __folio_memcg(folio); 7149 } 7150 7151 if (!memcg) 7152 return; 7153 7154 if (ug->memcg != memcg) { 7155 if (ug->memcg) { 7156 uncharge_batch(ug); 7157 uncharge_gather_clear(ug); 7158 } 7159 ug->memcg = memcg; 7160 ug->nid = folio_nid(folio); 7161 7162 /* pairs with css_put in uncharge_batch */ 7163 css_get(&memcg->css); 7164 } 7165 7166 nr_pages = folio_nr_pages(folio); 7167 7168 if (folio_memcg_kmem(folio)) { 7169 ug->nr_memory += nr_pages; 7170 ug->nr_kmem += nr_pages; 7171 7172 folio->memcg_data = 0; 7173 obj_cgroup_put(objcg); 7174 } else { 7175 /* LRU pages aren't accounted at the root level */ 7176 if (!mem_cgroup_is_root(memcg)) 7177 ug->nr_memory += nr_pages; 7178 ug->pgpgout++; 7179 7180 folio->memcg_data = 0; 7181 } 7182 7183 css_put(&memcg->css); 7184 } 7185 7186 void __mem_cgroup_uncharge(struct folio *folio) 7187 { 7188 struct uncharge_gather ug; 7189 7190 /* Don't touch folio->lru of any random page, pre-check: */ 7191 if (!folio_memcg(folio)) 7192 return; 7193 7194 uncharge_gather_clear(&ug); 7195 uncharge_folio(folio, &ug); 7196 uncharge_batch(&ug); 7197 } 7198 7199 /** 7200 * __mem_cgroup_uncharge_list - uncharge a list of page 7201 * @page_list: list of pages to uncharge 7202 * 7203 * Uncharge a list of pages previously charged with 7204 * __mem_cgroup_charge(). 7205 */ 7206 void __mem_cgroup_uncharge_list(struct list_head *page_list) 7207 { 7208 struct uncharge_gather ug; 7209 struct folio *folio; 7210 7211 uncharge_gather_clear(&ug); 7212 list_for_each_entry(folio, page_list, lru) 7213 uncharge_folio(folio, &ug); 7214 if (ug.memcg) 7215 uncharge_batch(&ug); 7216 } 7217 7218 /** 7219 * mem_cgroup_migrate - Charge a folio's replacement. 7220 * @old: Currently circulating folio. 7221 * @new: Replacement folio. 7222 * 7223 * Charge @new as a replacement folio for @old. @old will 7224 * be uncharged upon free. 7225 * 7226 * Both folios must be locked, @new->mapping must be set up. 7227 */ 7228 void mem_cgroup_migrate(struct folio *old, struct folio *new) 7229 { 7230 struct mem_cgroup *memcg; 7231 long nr_pages = folio_nr_pages(new); 7232 unsigned long flags; 7233 7234 VM_BUG_ON_FOLIO(!folio_test_locked(old), old); 7235 VM_BUG_ON_FOLIO(!folio_test_locked(new), new); 7236 VM_BUG_ON_FOLIO(folio_test_anon(old) != folio_test_anon(new), new); 7237 VM_BUG_ON_FOLIO(folio_nr_pages(old) != nr_pages, new); 7238 7239 if (mem_cgroup_disabled()) 7240 return; 7241 7242 /* Page cache replacement: new folio already charged? */ 7243 if (folio_memcg(new)) 7244 return; 7245 7246 memcg = folio_memcg(old); 7247 VM_WARN_ON_ONCE_FOLIO(!memcg, old); 7248 if (!memcg) 7249 return; 7250 7251 /* Force-charge the new page. The old one will be freed soon */ 7252 if (!mem_cgroup_is_root(memcg)) { 7253 page_counter_charge(&memcg->memory, nr_pages); 7254 if (do_memsw_account()) 7255 page_counter_charge(&memcg->memsw, nr_pages); 7256 } 7257 7258 css_get(&memcg->css); 7259 commit_charge(new, memcg); 7260 7261 local_irq_save(flags); 7262 mem_cgroup_charge_statistics(memcg, nr_pages); 7263 memcg_check_events(memcg, folio_nid(new)); 7264 local_irq_restore(flags); 7265 } 7266 7267 DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key); 7268 EXPORT_SYMBOL(memcg_sockets_enabled_key); 7269 7270 void mem_cgroup_sk_alloc(struct sock *sk) 7271 { 7272 struct mem_cgroup *memcg; 7273 7274 if (!mem_cgroup_sockets_enabled) 7275 return; 7276 7277 /* Do not associate the sock with unrelated interrupted task's memcg. */ 7278 if (!in_task()) 7279 return; 7280 7281 rcu_read_lock(); 7282 memcg = mem_cgroup_from_task(current); 7283 if (mem_cgroup_is_root(memcg)) 7284 goto out; 7285 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active) 7286 goto out; 7287 if (css_tryget(&memcg->css)) 7288 sk->sk_memcg = memcg; 7289 out: 7290 rcu_read_unlock(); 7291 } 7292 7293 void mem_cgroup_sk_free(struct sock *sk) 7294 { 7295 if (sk->sk_memcg) 7296 css_put(&sk->sk_memcg->css); 7297 } 7298 7299 /** 7300 * mem_cgroup_charge_skmem - charge socket memory 7301 * @memcg: memcg to charge 7302 * @nr_pages: number of pages to charge 7303 * @gfp_mask: reclaim mode 7304 * 7305 * Charges @nr_pages to @memcg. Returns %true if the charge fit within 7306 * @memcg's configured limit, %false if it doesn't. 7307 */ 7308 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages, 7309 gfp_t gfp_mask) 7310 { 7311 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) { 7312 struct page_counter *fail; 7313 7314 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) { 7315 memcg->tcpmem_pressure = 0; 7316 return true; 7317 } 7318 memcg->tcpmem_pressure = 1; 7319 if (gfp_mask & __GFP_NOFAIL) { 7320 page_counter_charge(&memcg->tcpmem, nr_pages); 7321 return true; 7322 } 7323 return false; 7324 } 7325 7326 if (try_charge(memcg, gfp_mask, nr_pages) == 0) { 7327 mod_memcg_state(memcg, MEMCG_SOCK, nr_pages); 7328 return true; 7329 } 7330 7331 return false; 7332 } 7333 7334 /** 7335 * mem_cgroup_uncharge_skmem - uncharge socket memory 7336 * @memcg: memcg to uncharge 7337 * @nr_pages: number of pages to uncharge 7338 */ 7339 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages) 7340 { 7341 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) { 7342 page_counter_uncharge(&memcg->tcpmem, nr_pages); 7343 return; 7344 } 7345 7346 mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages); 7347 7348 refill_stock(memcg, nr_pages); 7349 } 7350 7351 static int __init cgroup_memory(char *s) 7352 { 7353 char *token; 7354 7355 while ((token = strsep(&s, ",")) != NULL) { 7356 if (!*token) 7357 continue; 7358 if (!strcmp(token, "nosocket")) 7359 cgroup_memory_nosocket = true; 7360 if (!strcmp(token, "nokmem")) 7361 cgroup_memory_nokmem = true; 7362 if (!strcmp(token, "nobpf")) 7363 cgroup_memory_nobpf = true; 7364 } 7365 return 1; 7366 } 7367 __setup("cgroup.memory=", cgroup_memory); 7368 7369 /* 7370 * subsys_initcall() for memory controller. 7371 * 7372 * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this 7373 * context because of lock dependencies (cgroup_lock -> cpu hotplug) but 7374 * basically everything that doesn't depend on a specific mem_cgroup structure 7375 * should be initialized from here. 7376 */ 7377 static int __init mem_cgroup_init(void) 7378 { 7379 int cpu, node; 7380 7381 /* 7382 * Currently s32 type (can refer to struct batched_lruvec_stat) is 7383 * used for per-memcg-per-cpu caching of per-node statistics. In order 7384 * to work fine, we should make sure that the overfill threshold can't 7385 * exceed S32_MAX / PAGE_SIZE. 7386 */ 7387 BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S32_MAX / PAGE_SIZE); 7388 7389 cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL, 7390 memcg_hotplug_cpu_dead); 7391 7392 for_each_possible_cpu(cpu) 7393 INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work, 7394 drain_local_stock); 7395 7396 for_each_node(node) { 7397 struct mem_cgroup_tree_per_node *rtpn; 7398 7399 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, node); 7400 7401 rtpn->rb_root = RB_ROOT; 7402 rtpn->rb_rightmost = NULL; 7403 spin_lock_init(&rtpn->lock); 7404 soft_limit_tree.rb_tree_per_node[node] = rtpn; 7405 } 7406 7407 return 0; 7408 } 7409 subsys_initcall(mem_cgroup_init); 7410 7411 #ifdef CONFIG_SWAP 7412 static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg) 7413 { 7414 while (!refcount_inc_not_zero(&memcg->id.ref)) { 7415 /* 7416 * The root cgroup cannot be destroyed, so it's refcount must 7417 * always be >= 1. 7418 */ 7419 if (WARN_ON_ONCE(mem_cgroup_is_root(memcg))) { 7420 VM_BUG_ON(1); 7421 break; 7422 } 7423 memcg = parent_mem_cgroup(memcg); 7424 if (!memcg) 7425 memcg = root_mem_cgroup; 7426 } 7427 return memcg; 7428 } 7429 7430 /** 7431 * mem_cgroup_swapout - transfer a memsw charge to swap 7432 * @folio: folio whose memsw charge to transfer 7433 * @entry: swap entry to move the charge to 7434 * 7435 * Transfer the memsw charge of @folio to @entry. 7436 */ 7437 void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry) 7438 { 7439 struct mem_cgroup *memcg, *swap_memcg; 7440 unsigned int nr_entries; 7441 unsigned short oldid; 7442 7443 VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); 7444 VM_BUG_ON_FOLIO(folio_ref_count(folio), folio); 7445 7446 if (mem_cgroup_disabled()) 7447 return; 7448 7449 if (!do_memsw_account()) 7450 return; 7451 7452 memcg = folio_memcg(folio); 7453 7454 VM_WARN_ON_ONCE_FOLIO(!memcg, folio); 7455 if (!memcg) 7456 return; 7457 7458 /* 7459 * In case the memcg owning these pages has been offlined and doesn't 7460 * have an ID allocated to it anymore, charge the closest online 7461 * ancestor for the swap instead and transfer the memory+swap charge. 7462 */ 7463 swap_memcg = mem_cgroup_id_get_online(memcg); 7464 nr_entries = folio_nr_pages(folio); 7465 /* Get references for the tail pages, too */ 7466 if (nr_entries > 1) 7467 mem_cgroup_id_get_many(swap_memcg, nr_entries - 1); 7468 oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg), 7469 nr_entries); 7470 VM_BUG_ON_FOLIO(oldid, folio); 7471 mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries); 7472 7473 folio->memcg_data = 0; 7474 7475 if (!mem_cgroup_is_root(memcg)) 7476 page_counter_uncharge(&memcg->memory, nr_entries); 7477 7478 if (memcg != swap_memcg) { 7479 if (!mem_cgroup_is_root(swap_memcg)) 7480 page_counter_charge(&swap_memcg->memsw, nr_entries); 7481 page_counter_uncharge(&memcg->memsw, nr_entries); 7482 } 7483 7484 /* 7485 * Interrupts should be disabled here because the caller holds the 7486 * i_pages lock which is taken with interrupts-off. It is 7487 * important here to have the interrupts disabled because it is the 7488 * only synchronisation we have for updating the per-CPU variables. 7489 */ 7490 memcg_stats_lock(); 7491 mem_cgroup_charge_statistics(memcg, -nr_entries); 7492 memcg_stats_unlock(); 7493 memcg_check_events(memcg, folio_nid(folio)); 7494 7495 css_put(&memcg->css); 7496 } 7497 7498 /** 7499 * __mem_cgroup_try_charge_swap - try charging swap space for a folio 7500 * @folio: folio being added to swap 7501 * @entry: swap entry to charge 7502 * 7503 * Try to charge @folio's memcg for the swap space at @entry. 7504 * 7505 * Returns 0 on success, -ENOMEM on failure. 7506 */ 7507 int __mem_cgroup_try_charge_swap(struct folio *folio, swp_entry_t entry) 7508 { 7509 unsigned int nr_pages = folio_nr_pages(folio); 7510 struct page_counter *counter; 7511 struct mem_cgroup *memcg; 7512 unsigned short oldid; 7513 7514 if (do_memsw_account()) 7515 return 0; 7516 7517 memcg = folio_memcg(folio); 7518 7519 VM_WARN_ON_ONCE_FOLIO(!memcg, folio); 7520 if (!memcg) 7521 return 0; 7522 7523 if (!entry.val) { 7524 memcg_memory_event(memcg, MEMCG_SWAP_FAIL); 7525 return 0; 7526 } 7527 7528 memcg = mem_cgroup_id_get_online(memcg); 7529 7530 if (!mem_cgroup_is_root(memcg) && 7531 !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) { 7532 memcg_memory_event(memcg, MEMCG_SWAP_MAX); 7533 memcg_memory_event(memcg, MEMCG_SWAP_FAIL); 7534 mem_cgroup_id_put(memcg); 7535 return -ENOMEM; 7536 } 7537 7538 /* Get references for the tail pages, too */ 7539 if (nr_pages > 1) 7540 mem_cgroup_id_get_many(memcg, nr_pages - 1); 7541 oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages); 7542 VM_BUG_ON_FOLIO(oldid, folio); 7543 mod_memcg_state(memcg, MEMCG_SWAP, nr_pages); 7544 7545 return 0; 7546 } 7547 7548 /** 7549 * __mem_cgroup_uncharge_swap - uncharge swap space 7550 * @entry: swap entry to uncharge 7551 * @nr_pages: the amount of swap space to uncharge 7552 */ 7553 void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages) 7554 { 7555 struct mem_cgroup *memcg; 7556 unsigned short id; 7557 7558 id = swap_cgroup_record(entry, 0, nr_pages); 7559 rcu_read_lock(); 7560 memcg = mem_cgroup_from_id(id); 7561 if (memcg) { 7562 if (!mem_cgroup_is_root(memcg)) { 7563 if (do_memsw_account()) 7564 page_counter_uncharge(&memcg->memsw, nr_pages); 7565 else 7566 page_counter_uncharge(&memcg->swap, nr_pages); 7567 } 7568 mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages); 7569 mem_cgroup_id_put_many(memcg, nr_pages); 7570 } 7571 rcu_read_unlock(); 7572 } 7573 7574 long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg) 7575 { 7576 long nr_swap_pages = get_nr_swap_pages(); 7577 7578 if (mem_cgroup_disabled() || do_memsw_account()) 7579 return nr_swap_pages; 7580 for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) 7581 nr_swap_pages = min_t(long, nr_swap_pages, 7582 READ_ONCE(memcg->swap.max) - 7583 page_counter_read(&memcg->swap)); 7584 return nr_swap_pages; 7585 } 7586 7587 bool mem_cgroup_swap_full(struct folio *folio) 7588 { 7589 struct mem_cgroup *memcg; 7590 7591 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 7592 7593 if (vm_swap_full()) 7594 return true; 7595 if (do_memsw_account()) 7596 return false; 7597 7598 memcg = folio_memcg(folio); 7599 if (!memcg) 7600 return false; 7601 7602 for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) { 7603 unsigned long usage = page_counter_read(&memcg->swap); 7604 7605 if (usage * 2 >= READ_ONCE(memcg->swap.high) || 7606 usage * 2 >= READ_ONCE(memcg->swap.max)) 7607 return true; 7608 } 7609 7610 return false; 7611 } 7612 7613 static int __init setup_swap_account(char *s) 7614 { 7615 pr_warn_once("The swapaccount= commandline option is deprecated. " 7616 "Please report your usecase to linux-mm@kvack.org if you " 7617 "depend on this functionality.\n"); 7618 return 1; 7619 } 7620 __setup("swapaccount=", setup_swap_account); 7621 7622 static u64 swap_current_read(struct cgroup_subsys_state *css, 7623 struct cftype *cft) 7624 { 7625 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 7626 7627 return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE; 7628 } 7629 7630 static u64 swap_peak_read(struct cgroup_subsys_state *css, 7631 struct cftype *cft) 7632 { 7633 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 7634 7635 return (u64)memcg->swap.watermark * PAGE_SIZE; 7636 } 7637 7638 static int swap_high_show(struct seq_file *m, void *v) 7639 { 7640 return seq_puts_memcg_tunable(m, 7641 READ_ONCE(mem_cgroup_from_seq(m)->swap.high)); 7642 } 7643 7644 static ssize_t swap_high_write(struct kernfs_open_file *of, 7645 char *buf, size_t nbytes, loff_t off) 7646 { 7647 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 7648 unsigned long high; 7649 int err; 7650 7651 buf = strstrip(buf); 7652 err = page_counter_memparse(buf, "max", &high); 7653 if (err) 7654 return err; 7655 7656 page_counter_set_high(&memcg->swap, high); 7657 7658 return nbytes; 7659 } 7660 7661 static int swap_max_show(struct seq_file *m, void *v) 7662 { 7663 return seq_puts_memcg_tunable(m, 7664 READ_ONCE(mem_cgroup_from_seq(m)->swap.max)); 7665 } 7666 7667 static ssize_t swap_max_write(struct kernfs_open_file *of, 7668 char *buf, size_t nbytes, loff_t off) 7669 { 7670 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 7671 unsigned long max; 7672 int err; 7673 7674 buf = strstrip(buf); 7675 err = page_counter_memparse(buf, "max", &max); 7676 if (err) 7677 return err; 7678 7679 xchg(&memcg->swap.max, max); 7680 7681 return nbytes; 7682 } 7683 7684 static int swap_events_show(struct seq_file *m, void *v) 7685 { 7686 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 7687 7688 seq_printf(m, "high %lu\n", 7689 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_HIGH])); 7690 seq_printf(m, "max %lu\n", 7691 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX])); 7692 seq_printf(m, "fail %lu\n", 7693 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL])); 7694 7695 return 0; 7696 } 7697 7698 static struct cftype swap_files[] = { 7699 { 7700 .name = "swap.current", 7701 .flags = CFTYPE_NOT_ON_ROOT, 7702 .read_u64 = swap_current_read, 7703 }, 7704 { 7705 .name = "swap.high", 7706 .flags = CFTYPE_NOT_ON_ROOT, 7707 .seq_show = swap_high_show, 7708 .write = swap_high_write, 7709 }, 7710 { 7711 .name = "swap.max", 7712 .flags = CFTYPE_NOT_ON_ROOT, 7713 .seq_show = swap_max_show, 7714 .write = swap_max_write, 7715 }, 7716 { 7717 .name = "swap.peak", 7718 .flags = CFTYPE_NOT_ON_ROOT, 7719 .read_u64 = swap_peak_read, 7720 }, 7721 { 7722 .name = "swap.events", 7723 .flags = CFTYPE_NOT_ON_ROOT, 7724 .file_offset = offsetof(struct mem_cgroup, swap_events_file), 7725 .seq_show = swap_events_show, 7726 }, 7727 { } /* terminate */ 7728 }; 7729 7730 static struct cftype memsw_files[] = { 7731 { 7732 .name = "memsw.usage_in_bytes", 7733 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE), 7734 .read_u64 = mem_cgroup_read_u64, 7735 }, 7736 { 7737 .name = "memsw.max_usage_in_bytes", 7738 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE), 7739 .write = mem_cgroup_reset, 7740 .read_u64 = mem_cgroup_read_u64, 7741 }, 7742 { 7743 .name = "memsw.limit_in_bytes", 7744 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT), 7745 .write = mem_cgroup_write, 7746 .read_u64 = mem_cgroup_read_u64, 7747 }, 7748 { 7749 .name = "memsw.failcnt", 7750 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT), 7751 .write = mem_cgroup_reset, 7752 .read_u64 = mem_cgroup_read_u64, 7753 }, 7754 { }, /* terminate */ 7755 }; 7756 7757 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP) 7758 /** 7759 * obj_cgroup_may_zswap - check if this cgroup can zswap 7760 * @objcg: the object cgroup 7761 * 7762 * Check if the hierarchical zswap limit has been reached. 7763 * 7764 * This doesn't check for specific headroom, and it is not atomic 7765 * either. But with zswap, the size of the allocation is only known 7766 * once compression has occured, and this optimistic pre-check avoids 7767 * spending cycles on compression when there is already no room left 7768 * or zswap is disabled altogether somewhere in the hierarchy. 7769 */ 7770 bool obj_cgroup_may_zswap(struct obj_cgroup *objcg) 7771 { 7772 struct mem_cgroup *memcg, *original_memcg; 7773 bool ret = true; 7774 7775 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) 7776 return true; 7777 7778 original_memcg = get_mem_cgroup_from_objcg(objcg); 7779 for (memcg = original_memcg; !mem_cgroup_is_root(memcg); 7780 memcg = parent_mem_cgroup(memcg)) { 7781 unsigned long max = READ_ONCE(memcg->zswap_max); 7782 unsigned long pages; 7783 7784 if (max == PAGE_COUNTER_MAX) 7785 continue; 7786 if (max == 0) { 7787 ret = false; 7788 break; 7789 } 7790 7791 cgroup_rstat_flush(memcg->css.cgroup); 7792 pages = memcg_page_state(memcg, MEMCG_ZSWAP_B) / PAGE_SIZE; 7793 if (pages < max) 7794 continue; 7795 ret = false; 7796 break; 7797 } 7798 mem_cgroup_put(original_memcg); 7799 return ret; 7800 } 7801 7802 /** 7803 * obj_cgroup_charge_zswap - charge compression backend memory 7804 * @objcg: the object cgroup 7805 * @size: size of compressed object 7806 * 7807 * This forces the charge after obj_cgroup_may_zswap() allowed 7808 * compression and storage in zwap for this cgroup to go ahead. 7809 */ 7810 void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size) 7811 { 7812 struct mem_cgroup *memcg; 7813 7814 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) 7815 return; 7816 7817 VM_WARN_ON_ONCE(!(current->flags & PF_MEMALLOC)); 7818 7819 /* PF_MEMALLOC context, charging must succeed */ 7820 if (obj_cgroup_charge(objcg, GFP_KERNEL, size)) 7821 VM_WARN_ON_ONCE(1); 7822 7823 rcu_read_lock(); 7824 memcg = obj_cgroup_memcg(objcg); 7825 mod_memcg_state(memcg, MEMCG_ZSWAP_B, size); 7826 mod_memcg_state(memcg, MEMCG_ZSWAPPED, 1); 7827 rcu_read_unlock(); 7828 } 7829 7830 /** 7831 * obj_cgroup_uncharge_zswap - uncharge compression backend memory 7832 * @objcg: the object cgroup 7833 * @size: size of compressed object 7834 * 7835 * Uncharges zswap memory on page in. 7836 */ 7837 void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size) 7838 { 7839 struct mem_cgroup *memcg; 7840 7841 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) 7842 return; 7843 7844 obj_cgroup_uncharge(objcg, size); 7845 7846 rcu_read_lock(); 7847 memcg = obj_cgroup_memcg(objcg); 7848 mod_memcg_state(memcg, MEMCG_ZSWAP_B, -size); 7849 mod_memcg_state(memcg, MEMCG_ZSWAPPED, -1); 7850 rcu_read_unlock(); 7851 } 7852 7853 static u64 zswap_current_read(struct cgroup_subsys_state *css, 7854 struct cftype *cft) 7855 { 7856 cgroup_rstat_flush(css->cgroup); 7857 return memcg_page_state(mem_cgroup_from_css(css), MEMCG_ZSWAP_B); 7858 } 7859 7860 static int zswap_max_show(struct seq_file *m, void *v) 7861 { 7862 return seq_puts_memcg_tunable(m, 7863 READ_ONCE(mem_cgroup_from_seq(m)->zswap_max)); 7864 } 7865 7866 static ssize_t zswap_max_write(struct kernfs_open_file *of, 7867 char *buf, size_t nbytes, loff_t off) 7868 { 7869 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 7870 unsigned long max; 7871 int err; 7872 7873 buf = strstrip(buf); 7874 err = page_counter_memparse(buf, "max", &max); 7875 if (err) 7876 return err; 7877 7878 xchg(&memcg->zswap_max, max); 7879 7880 return nbytes; 7881 } 7882 7883 static struct cftype zswap_files[] = { 7884 { 7885 .name = "zswap.current", 7886 .flags = CFTYPE_NOT_ON_ROOT, 7887 .read_u64 = zswap_current_read, 7888 }, 7889 { 7890 .name = "zswap.max", 7891 .flags = CFTYPE_NOT_ON_ROOT, 7892 .seq_show = zswap_max_show, 7893 .write = zswap_max_write, 7894 }, 7895 { } /* terminate */ 7896 }; 7897 #endif /* CONFIG_MEMCG_KMEM && CONFIG_ZSWAP */ 7898 7899 static int __init mem_cgroup_swap_init(void) 7900 { 7901 if (mem_cgroup_disabled()) 7902 return 0; 7903 7904 WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, swap_files)); 7905 WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys, memsw_files)); 7906 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP) 7907 WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, zswap_files)); 7908 #endif 7909 return 0; 7910 } 7911 subsys_initcall(mem_cgroup_swap_init); 7912 7913 #endif /* CONFIG_SWAP */ 7914