1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* memcontrol.c - Memory Controller 3 * 4 * Copyright IBM Corporation, 2007 5 * Author Balbir Singh <balbir@linux.vnet.ibm.com> 6 * 7 * Copyright 2007 OpenVZ SWsoft Inc 8 * Author: Pavel Emelianov <xemul@openvz.org> 9 * 10 * Memory thresholds 11 * Copyright (C) 2009 Nokia Corporation 12 * Author: Kirill A. Shutemov 13 * 14 * Kernel Memory Controller 15 * Copyright (C) 2012 Parallels Inc. and Google Inc. 16 * Authors: Glauber Costa and Suleiman Souhlal 17 * 18 * Native page reclaim 19 * Charge lifetime sanitation 20 * Lockless page tracking & accounting 21 * Unified hierarchy configuration model 22 * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner 23 * 24 * Per memcg lru locking 25 * Copyright (C) 2020 Alibaba, Inc, Alex Shi 26 */ 27 28 #include <linux/page_counter.h> 29 #include <linux/memcontrol.h> 30 #include <linux/cgroup.h> 31 #include <linux/pagewalk.h> 32 #include <linux/sched/mm.h> 33 #include <linux/shmem_fs.h> 34 #include <linux/hugetlb.h> 35 #include <linux/pagemap.h> 36 #include <linux/vm_event_item.h> 37 #include <linux/smp.h> 38 #include <linux/page-flags.h> 39 #include <linux/backing-dev.h> 40 #include <linux/bit_spinlock.h> 41 #include <linux/rcupdate.h> 42 #include <linux/limits.h> 43 #include <linux/export.h> 44 #include <linux/mutex.h> 45 #include <linux/rbtree.h> 46 #include <linux/slab.h> 47 #include <linux/swap.h> 48 #include <linux/swapops.h> 49 #include <linux/spinlock.h> 50 #include <linux/eventfd.h> 51 #include <linux/poll.h> 52 #include <linux/sort.h> 53 #include <linux/fs.h> 54 #include <linux/seq_file.h> 55 #include <linux/vmpressure.h> 56 #include <linux/mm_inline.h> 57 #include <linux/swap_cgroup.h> 58 #include <linux/cpu.h> 59 #include <linux/oom.h> 60 #include <linux/lockdep.h> 61 #include <linux/file.h> 62 #include <linux/tracehook.h> 63 #include <linux/psi.h> 64 #include <linux/seq_buf.h> 65 #include "internal.h" 66 #include <net/sock.h> 67 #include <net/ip.h> 68 #include "slab.h" 69 70 #include <linux/uaccess.h> 71 72 #include <trace/events/vmscan.h> 73 74 struct cgroup_subsys memory_cgrp_subsys __read_mostly; 75 EXPORT_SYMBOL(memory_cgrp_subsys); 76 77 struct mem_cgroup *root_mem_cgroup __read_mostly; 78 79 /* Active memory cgroup to use from an interrupt context */ 80 DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg); 81 EXPORT_PER_CPU_SYMBOL_GPL(int_active_memcg); 82 83 /* Socket memory accounting disabled? */ 84 static bool cgroup_memory_nosocket __ro_after_init; 85 86 /* Kernel memory accounting disabled? */ 87 bool cgroup_memory_nokmem __ro_after_init; 88 89 /* Whether the swap controller is active */ 90 #ifdef CONFIG_MEMCG_SWAP 91 bool cgroup_memory_noswap __ro_after_init; 92 #else 93 #define cgroup_memory_noswap 1 94 #endif 95 96 #ifdef CONFIG_CGROUP_WRITEBACK 97 static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq); 98 #endif 99 100 /* Whether legacy memory+swap accounting is active */ 101 static bool do_memsw_account(void) 102 { 103 return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_noswap; 104 } 105 106 /* memcg and lruvec stats flushing */ 107 static void flush_memcg_stats_dwork(struct work_struct *w); 108 static DECLARE_DEFERRABLE_WORK(stats_flush_dwork, flush_memcg_stats_dwork); 109 static DEFINE_SPINLOCK(stats_flush_lock); 110 111 #define THRESHOLDS_EVENTS_TARGET 128 112 #define SOFTLIMIT_EVENTS_TARGET 1024 113 114 /* 115 * Cgroups above their limits are maintained in a RB-Tree, independent of 116 * their hierarchy representation 117 */ 118 119 struct mem_cgroup_tree_per_node { 120 struct rb_root rb_root; 121 struct rb_node *rb_rightmost; 122 spinlock_t lock; 123 }; 124 125 struct mem_cgroup_tree { 126 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES]; 127 }; 128 129 static struct mem_cgroup_tree soft_limit_tree __read_mostly; 130 131 /* for OOM */ 132 struct mem_cgroup_eventfd_list { 133 struct list_head list; 134 struct eventfd_ctx *eventfd; 135 }; 136 137 /* 138 * cgroup_event represents events which userspace want to receive. 139 */ 140 struct mem_cgroup_event { 141 /* 142 * memcg which the event belongs to. 143 */ 144 struct mem_cgroup *memcg; 145 /* 146 * eventfd to signal userspace about the event. 147 */ 148 struct eventfd_ctx *eventfd; 149 /* 150 * Each of these stored in a list by the cgroup. 151 */ 152 struct list_head list; 153 /* 154 * register_event() callback will be used to add new userspace 155 * waiter for changes related to this event. Use eventfd_signal() 156 * on eventfd to send notification to userspace. 157 */ 158 int (*register_event)(struct mem_cgroup *memcg, 159 struct eventfd_ctx *eventfd, const char *args); 160 /* 161 * unregister_event() callback will be called when userspace closes 162 * the eventfd or on cgroup removing. This callback must be set, 163 * if you want provide notification functionality. 164 */ 165 void (*unregister_event)(struct mem_cgroup *memcg, 166 struct eventfd_ctx *eventfd); 167 /* 168 * All fields below needed to unregister event when 169 * userspace closes eventfd. 170 */ 171 poll_table pt; 172 wait_queue_head_t *wqh; 173 wait_queue_entry_t wait; 174 struct work_struct remove; 175 }; 176 177 static void mem_cgroup_threshold(struct mem_cgroup *memcg); 178 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg); 179 180 /* Stuffs for move charges at task migration. */ 181 /* 182 * Types of charges to be moved. 183 */ 184 #define MOVE_ANON 0x1U 185 #define MOVE_FILE 0x2U 186 #define MOVE_MASK (MOVE_ANON | MOVE_FILE) 187 188 /* "mc" and its members are protected by cgroup_mutex */ 189 static struct move_charge_struct { 190 spinlock_t lock; /* for from, to */ 191 struct mm_struct *mm; 192 struct mem_cgroup *from; 193 struct mem_cgroup *to; 194 unsigned long flags; 195 unsigned long precharge; 196 unsigned long moved_charge; 197 unsigned long moved_swap; 198 struct task_struct *moving_task; /* a task moving charges */ 199 wait_queue_head_t waitq; /* a waitq for other context */ 200 } mc = { 201 .lock = __SPIN_LOCK_UNLOCKED(mc.lock), 202 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq), 203 }; 204 205 /* 206 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft 207 * limit reclaim to prevent infinite loops, if they ever occur. 208 */ 209 #define MEM_CGROUP_MAX_RECLAIM_LOOPS 100 210 #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2 211 212 /* for encoding cft->private value on file */ 213 enum res_type { 214 _MEM, 215 _MEMSWAP, 216 _OOM_TYPE, 217 _KMEM, 218 _TCP, 219 }; 220 221 #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val)) 222 #define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff) 223 #define MEMFILE_ATTR(val) ((val) & 0xffff) 224 /* Used for OOM notifier */ 225 #define OOM_CONTROL (0) 226 227 /* 228 * Iteration constructs for visiting all cgroups (under a tree). If 229 * loops are exited prematurely (break), mem_cgroup_iter_break() must 230 * be used for reference counting. 231 */ 232 #define for_each_mem_cgroup_tree(iter, root) \ 233 for (iter = mem_cgroup_iter(root, NULL, NULL); \ 234 iter != NULL; \ 235 iter = mem_cgroup_iter(root, iter, NULL)) 236 237 #define for_each_mem_cgroup(iter) \ 238 for (iter = mem_cgroup_iter(NULL, NULL, NULL); \ 239 iter != NULL; \ 240 iter = mem_cgroup_iter(NULL, iter, NULL)) 241 242 static inline bool should_force_charge(void) 243 { 244 return tsk_is_oom_victim(current) || fatal_signal_pending(current) || 245 (current->flags & PF_EXITING); 246 } 247 248 /* Some nice accessors for the vmpressure. */ 249 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg) 250 { 251 if (!memcg) 252 memcg = root_mem_cgroup; 253 return &memcg->vmpressure; 254 } 255 256 struct mem_cgroup *vmpressure_to_memcg(struct vmpressure *vmpr) 257 { 258 return container_of(vmpr, struct mem_cgroup, vmpressure); 259 } 260 261 #ifdef CONFIG_MEMCG_KMEM 262 extern spinlock_t css_set_lock; 263 264 bool mem_cgroup_kmem_disabled(void) 265 { 266 return cgroup_memory_nokmem; 267 } 268 269 static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg, 270 unsigned int nr_pages); 271 272 static void obj_cgroup_release(struct percpu_ref *ref) 273 { 274 struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt); 275 unsigned int nr_bytes; 276 unsigned int nr_pages; 277 unsigned long flags; 278 279 /* 280 * At this point all allocated objects are freed, and 281 * objcg->nr_charged_bytes can't have an arbitrary byte value. 282 * However, it can be PAGE_SIZE or (x * PAGE_SIZE). 283 * 284 * The following sequence can lead to it: 285 * 1) CPU0: objcg == stock->cached_objcg 286 * 2) CPU1: we do a small allocation (e.g. 92 bytes), 287 * PAGE_SIZE bytes are charged 288 * 3) CPU1: a process from another memcg is allocating something, 289 * the stock if flushed, 290 * objcg->nr_charged_bytes = PAGE_SIZE - 92 291 * 5) CPU0: we do release this object, 292 * 92 bytes are added to stock->nr_bytes 293 * 6) CPU0: stock is flushed, 294 * 92 bytes are added to objcg->nr_charged_bytes 295 * 296 * In the result, nr_charged_bytes == PAGE_SIZE. 297 * This page will be uncharged in obj_cgroup_release(). 298 */ 299 nr_bytes = atomic_read(&objcg->nr_charged_bytes); 300 WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1)); 301 nr_pages = nr_bytes >> PAGE_SHIFT; 302 303 if (nr_pages) 304 obj_cgroup_uncharge_pages(objcg, nr_pages); 305 306 spin_lock_irqsave(&css_set_lock, flags); 307 list_del(&objcg->list); 308 spin_unlock_irqrestore(&css_set_lock, flags); 309 310 percpu_ref_exit(ref); 311 kfree_rcu(objcg, rcu); 312 } 313 314 static struct obj_cgroup *obj_cgroup_alloc(void) 315 { 316 struct obj_cgroup *objcg; 317 int ret; 318 319 objcg = kzalloc(sizeof(struct obj_cgroup), GFP_KERNEL); 320 if (!objcg) 321 return NULL; 322 323 ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0, 324 GFP_KERNEL); 325 if (ret) { 326 kfree(objcg); 327 return NULL; 328 } 329 INIT_LIST_HEAD(&objcg->list); 330 return objcg; 331 } 332 333 static void memcg_reparent_objcgs(struct mem_cgroup *memcg, 334 struct mem_cgroup *parent) 335 { 336 struct obj_cgroup *objcg, *iter; 337 338 objcg = rcu_replace_pointer(memcg->objcg, NULL, true); 339 340 spin_lock_irq(&css_set_lock); 341 342 /* 1) Ready to reparent active objcg. */ 343 list_add(&objcg->list, &memcg->objcg_list); 344 /* 2) Reparent active objcg and already reparented objcgs to parent. */ 345 list_for_each_entry(iter, &memcg->objcg_list, list) 346 WRITE_ONCE(iter->memcg, parent); 347 /* 3) Move already reparented objcgs to the parent's list */ 348 list_splice(&memcg->objcg_list, &parent->objcg_list); 349 350 spin_unlock_irq(&css_set_lock); 351 352 percpu_ref_kill(&objcg->refcnt); 353 } 354 355 /* 356 * This will be used as a shrinker list's index. 357 * The main reason for not using cgroup id for this: 358 * this works better in sparse environments, where we have a lot of memcgs, 359 * but only a few kmem-limited. Or also, if we have, for instance, 200 360 * memcgs, and none but the 200th is kmem-limited, we'd have to have a 361 * 200 entry array for that. 362 * 363 * The current size of the caches array is stored in memcg_nr_cache_ids. It 364 * will double each time we have to increase it. 365 */ 366 static DEFINE_IDA(memcg_cache_ida); 367 int memcg_nr_cache_ids; 368 369 /* Protects memcg_nr_cache_ids */ 370 static DECLARE_RWSEM(memcg_cache_ids_sem); 371 372 void memcg_get_cache_ids(void) 373 { 374 down_read(&memcg_cache_ids_sem); 375 } 376 377 void memcg_put_cache_ids(void) 378 { 379 up_read(&memcg_cache_ids_sem); 380 } 381 382 /* 383 * MIN_SIZE is different than 1, because we would like to avoid going through 384 * the alloc/free process all the time. In a small machine, 4 kmem-limited 385 * cgroups is a reasonable guess. In the future, it could be a parameter or 386 * tunable, but that is strictly not necessary. 387 * 388 * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get 389 * this constant directly from cgroup, but it is understandable that this is 390 * better kept as an internal representation in cgroup.c. In any case, the 391 * cgrp_id space is not getting any smaller, and we don't have to necessarily 392 * increase ours as well if it increases. 393 */ 394 #define MEMCG_CACHES_MIN_SIZE 4 395 #define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX 396 397 /* 398 * A lot of the calls to the cache allocation functions are expected to be 399 * inlined by the compiler. Since the calls to memcg_slab_pre_alloc_hook() are 400 * conditional to this static branch, we'll have to allow modules that does 401 * kmem_cache_alloc and the such to see this symbol as well 402 */ 403 DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key); 404 EXPORT_SYMBOL(memcg_kmem_enabled_key); 405 #endif 406 407 /** 408 * mem_cgroup_css_from_page - css of the memcg associated with a page 409 * @page: page of interest 410 * 411 * If memcg is bound to the default hierarchy, css of the memcg associated 412 * with @page is returned. The returned css remains associated with @page 413 * until it is released. 414 * 415 * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup 416 * is returned. 417 */ 418 struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page) 419 { 420 struct mem_cgroup *memcg; 421 422 memcg = page_memcg(page); 423 424 if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys)) 425 memcg = root_mem_cgroup; 426 427 return &memcg->css; 428 } 429 430 /** 431 * page_cgroup_ino - return inode number of the memcg a page is charged to 432 * @page: the page 433 * 434 * Look up the closest online ancestor of the memory cgroup @page is charged to 435 * and return its inode number or 0 if @page is not charged to any cgroup. It 436 * is safe to call this function without holding a reference to @page. 437 * 438 * Note, this function is inherently racy, because there is nothing to prevent 439 * the cgroup inode from getting torn down and potentially reallocated a moment 440 * after page_cgroup_ino() returns, so it only should be used by callers that 441 * do not care (such as procfs interfaces). 442 */ 443 ino_t page_cgroup_ino(struct page *page) 444 { 445 struct mem_cgroup *memcg; 446 unsigned long ino = 0; 447 448 rcu_read_lock(); 449 memcg = page_memcg_check(page); 450 451 while (memcg && !(memcg->css.flags & CSS_ONLINE)) 452 memcg = parent_mem_cgroup(memcg); 453 if (memcg) 454 ino = cgroup_ino(memcg->css.cgroup); 455 rcu_read_unlock(); 456 return ino; 457 } 458 459 static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz, 460 struct mem_cgroup_tree_per_node *mctz, 461 unsigned long new_usage_in_excess) 462 { 463 struct rb_node **p = &mctz->rb_root.rb_node; 464 struct rb_node *parent = NULL; 465 struct mem_cgroup_per_node *mz_node; 466 bool rightmost = true; 467 468 if (mz->on_tree) 469 return; 470 471 mz->usage_in_excess = new_usage_in_excess; 472 if (!mz->usage_in_excess) 473 return; 474 while (*p) { 475 parent = *p; 476 mz_node = rb_entry(parent, struct mem_cgroup_per_node, 477 tree_node); 478 if (mz->usage_in_excess < mz_node->usage_in_excess) { 479 p = &(*p)->rb_left; 480 rightmost = false; 481 } else { 482 p = &(*p)->rb_right; 483 } 484 } 485 486 if (rightmost) 487 mctz->rb_rightmost = &mz->tree_node; 488 489 rb_link_node(&mz->tree_node, parent, p); 490 rb_insert_color(&mz->tree_node, &mctz->rb_root); 491 mz->on_tree = true; 492 } 493 494 static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz, 495 struct mem_cgroup_tree_per_node *mctz) 496 { 497 if (!mz->on_tree) 498 return; 499 500 if (&mz->tree_node == mctz->rb_rightmost) 501 mctz->rb_rightmost = rb_prev(&mz->tree_node); 502 503 rb_erase(&mz->tree_node, &mctz->rb_root); 504 mz->on_tree = false; 505 } 506 507 static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz, 508 struct mem_cgroup_tree_per_node *mctz) 509 { 510 unsigned long flags; 511 512 spin_lock_irqsave(&mctz->lock, flags); 513 __mem_cgroup_remove_exceeded(mz, mctz); 514 spin_unlock_irqrestore(&mctz->lock, flags); 515 } 516 517 static unsigned long soft_limit_excess(struct mem_cgroup *memcg) 518 { 519 unsigned long nr_pages = page_counter_read(&memcg->memory); 520 unsigned long soft_limit = READ_ONCE(memcg->soft_limit); 521 unsigned long excess = 0; 522 523 if (nr_pages > soft_limit) 524 excess = nr_pages - soft_limit; 525 526 return excess; 527 } 528 529 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, int nid) 530 { 531 unsigned long excess; 532 struct mem_cgroup_per_node *mz; 533 struct mem_cgroup_tree_per_node *mctz; 534 535 mctz = soft_limit_tree.rb_tree_per_node[nid]; 536 if (!mctz) 537 return; 538 /* 539 * Necessary to update all ancestors when hierarchy is used. 540 * because their event counter is not touched. 541 */ 542 for (; memcg; memcg = parent_mem_cgroup(memcg)) { 543 mz = memcg->nodeinfo[nid]; 544 excess = soft_limit_excess(memcg); 545 /* 546 * We have to update the tree if mz is on RB-tree or 547 * mem is over its softlimit. 548 */ 549 if (excess || mz->on_tree) { 550 unsigned long flags; 551 552 spin_lock_irqsave(&mctz->lock, flags); 553 /* if on-tree, remove it */ 554 if (mz->on_tree) 555 __mem_cgroup_remove_exceeded(mz, mctz); 556 /* 557 * Insert again. mz->usage_in_excess will be updated. 558 * If excess is 0, no tree ops. 559 */ 560 __mem_cgroup_insert_exceeded(mz, mctz, excess); 561 spin_unlock_irqrestore(&mctz->lock, flags); 562 } 563 } 564 } 565 566 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg) 567 { 568 struct mem_cgroup_tree_per_node *mctz; 569 struct mem_cgroup_per_node *mz; 570 int nid; 571 572 for_each_node(nid) { 573 mz = memcg->nodeinfo[nid]; 574 mctz = soft_limit_tree.rb_tree_per_node[nid]; 575 if (mctz) 576 mem_cgroup_remove_exceeded(mz, mctz); 577 } 578 } 579 580 static struct mem_cgroup_per_node * 581 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz) 582 { 583 struct mem_cgroup_per_node *mz; 584 585 retry: 586 mz = NULL; 587 if (!mctz->rb_rightmost) 588 goto done; /* Nothing to reclaim from */ 589 590 mz = rb_entry(mctz->rb_rightmost, 591 struct mem_cgroup_per_node, tree_node); 592 /* 593 * Remove the node now but someone else can add it back, 594 * we will to add it back at the end of reclaim to its correct 595 * position in the tree. 596 */ 597 __mem_cgroup_remove_exceeded(mz, mctz); 598 if (!soft_limit_excess(mz->memcg) || 599 !css_tryget(&mz->memcg->css)) 600 goto retry; 601 done: 602 return mz; 603 } 604 605 static struct mem_cgroup_per_node * 606 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz) 607 { 608 struct mem_cgroup_per_node *mz; 609 610 spin_lock_irq(&mctz->lock); 611 mz = __mem_cgroup_largest_soft_limit_node(mctz); 612 spin_unlock_irq(&mctz->lock); 613 return mz; 614 } 615 616 /** 617 * __mod_memcg_state - update cgroup memory statistics 618 * @memcg: the memory cgroup 619 * @idx: the stat item - can be enum memcg_stat_item or enum node_stat_item 620 * @val: delta to add to the counter, can be negative 621 */ 622 void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val) 623 { 624 if (mem_cgroup_disabled()) 625 return; 626 627 __this_cpu_add(memcg->vmstats_percpu->state[idx], val); 628 cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id()); 629 } 630 631 /* idx can be of type enum memcg_stat_item or node_stat_item. */ 632 static unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx) 633 { 634 long x = 0; 635 int cpu; 636 637 for_each_possible_cpu(cpu) 638 x += per_cpu(memcg->vmstats_percpu->state[idx], cpu); 639 #ifdef CONFIG_SMP 640 if (x < 0) 641 x = 0; 642 #endif 643 return x; 644 } 645 646 void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, 647 int val) 648 { 649 struct mem_cgroup_per_node *pn; 650 struct mem_cgroup *memcg; 651 652 pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); 653 memcg = pn->memcg; 654 655 /* Update memcg */ 656 __mod_memcg_state(memcg, idx, val); 657 658 /* Update lruvec */ 659 __this_cpu_add(pn->lruvec_stats_percpu->state[idx], val); 660 } 661 662 /** 663 * __mod_lruvec_state - update lruvec memory statistics 664 * @lruvec: the lruvec 665 * @idx: the stat item 666 * @val: delta to add to the counter, can be negative 667 * 668 * The lruvec is the intersection of the NUMA node and a cgroup. This 669 * function updates the all three counters that are affected by a 670 * change of state at this level: per-node, per-cgroup, per-lruvec. 671 */ 672 void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, 673 int val) 674 { 675 /* Update node */ 676 __mod_node_page_state(lruvec_pgdat(lruvec), idx, val); 677 678 /* Update memcg and lruvec */ 679 if (!mem_cgroup_disabled()) 680 __mod_memcg_lruvec_state(lruvec, idx, val); 681 } 682 683 void __mod_lruvec_page_state(struct page *page, enum node_stat_item idx, 684 int val) 685 { 686 struct page *head = compound_head(page); /* rmap on tail pages */ 687 struct mem_cgroup *memcg; 688 pg_data_t *pgdat = page_pgdat(page); 689 struct lruvec *lruvec; 690 691 rcu_read_lock(); 692 memcg = page_memcg(head); 693 /* Untracked pages have no memcg, no lruvec. Update only the node */ 694 if (!memcg) { 695 rcu_read_unlock(); 696 __mod_node_page_state(pgdat, idx, val); 697 return; 698 } 699 700 lruvec = mem_cgroup_lruvec(memcg, pgdat); 701 __mod_lruvec_state(lruvec, idx, val); 702 rcu_read_unlock(); 703 } 704 EXPORT_SYMBOL(__mod_lruvec_page_state); 705 706 void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val) 707 { 708 pg_data_t *pgdat = page_pgdat(virt_to_page(p)); 709 struct mem_cgroup *memcg; 710 struct lruvec *lruvec; 711 712 rcu_read_lock(); 713 memcg = mem_cgroup_from_obj(p); 714 715 /* 716 * Untracked pages have no memcg, no lruvec. Update only the 717 * node. If we reparent the slab objects to the root memcg, 718 * when we free the slab object, we need to update the per-memcg 719 * vmstats to keep it correct for the root memcg. 720 */ 721 if (!memcg) { 722 __mod_node_page_state(pgdat, idx, val); 723 } else { 724 lruvec = mem_cgroup_lruvec(memcg, pgdat); 725 __mod_lruvec_state(lruvec, idx, val); 726 } 727 rcu_read_unlock(); 728 } 729 730 /* 731 * mod_objcg_mlstate() may be called with irq enabled, so 732 * mod_memcg_lruvec_state() should be used. 733 */ 734 static inline void mod_objcg_mlstate(struct obj_cgroup *objcg, 735 struct pglist_data *pgdat, 736 enum node_stat_item idx, int nr) 737 { 738 struct mem_cgroup *memcg; 739 struct lruvec *lruvec; 740 741 rcu_read_lock(); 742 memcg = obj_cgroup_memcg(objcg); 743 lruvec = mem_cgroup_lruvec(memcg, pgdat); 744 mod_memcg_lruvec_state(lruvec, idx, nr); 745 rcu_read_unlock(); 746 } 747 748 /** 749 * __count_memcg_events - account VM events in a cgroup 750 * @memcg: the memory cgroup 751 * @idx: the event item 752 * @count: the number of events that occurred 753 */ 754 void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, 755 unsigned long count) 756 { 757 if (mem_cgroup_disabled()) 758 return; 759 760 __this_cpu_add(memcg->vmstats_percpu->events[idx], count); 761 cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id()); 762 } 763 764 static unsigned long memcg_events(struct mem_cgroup *memcg, int event) 765 { 766 return READ_ONCE(memcg->vmstats.events[event]); 767 } 768 769 static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event) 770 { 771 long x = 0; 772 int cpu; 773 774 for_each_possible_cpu(cpu) 775 x += per_cpu(memcg->vmstats_percpu->events[event], cpu); 776 return x; 777 } 778 779 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg, 780 int nr_pages) 781 { 782 /* pagein of a big page is an event. So, ignore page size */ 783 if (nr_pages > 0) 784 __count_memcg_events(memcg, PGPGIN, 1); 785 else { 786 __count_memcg_events(memcg, PGPGOUT, 1); 787 nr_pages = -nr_pages; /* for event */ 788 } 789 790 __this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages); 791 } 792 793 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg, 794 enum mem_cgroup_events_target target) 795 { 796 unsigned long val, next; 797 798 val = __this_cpu_read(memcg->vmstats_percpu->nr_page_events); 799 next = __this_cpu_read(memcg->vmstats_percpu->targets[target]); 800 /* from time_after() in jiffies.h */ 801 if ((long)(next - val) < 0) { 802 switch (target) { 803 case MEM_CGROUP_TARGET_THRESH: 804 next = val + THRESHOLDS_EVENTS_TARGET; 805 break; 806 case MEM_CGROUP_TARGET_SOFTLIMIT: 807 next = val + SOFTLIMIT_EVENTS_TARGET; 808 break; 809 default: 810 break; 811 } 812 __this_cpu_write(memcg->vmstats_percpu->targets[target], next); 813 return true; 814 } 815 return false; 816 } 817 818 /* 819 * Check events in order. 820 * 821 */ 822 static void memcg_check_events(struct mem_cgroup *memcg, int nid) 823 { 824 /* threshold event is triggered in finer grain than soft limit */ 825 if (unlikely(mem_cgroup_event_ratelimit(memcg, 826 MEM_CGROUP_TARGET_THRESH))) { 827 bool do_softlimit; 828 829 do_softlimit = mem_cgroup_event_ratelimit(memcg, 830 MEM_CGROUP_TARGET_SOFTLIMIT); 831 mem_cgroup_threshold(memcg); 832 if (unlikely(do_softlimit)) 833 mem_cgroup_update_tree(memcg, nid); 834 } 835 } 836 837 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) 838 { 839 /* 840 * mm_update_next_owner() may clear mm->owner to NULL 841 * if it races with swapoff, page migration, etc. 842 * So this can be called with p == NULL. 843 */ 844 if (unlikely(!p)) 845 return NULL; 846 847 return mem_cgroup_from_css(task_css(p, memory_cgrp_id)); 848 } 849 EXPORT_SYMBOL(mem_cgroup_from_task); 850 851 static __always_inline struct mem_cgroup *active_memcg(void) 852 { 853 if (!in_task()) 854 return this_cpu_read(int_active_memcg); 855 else 856 return current->active_memcg; 857 } 858 859 /** 860 * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg. 861 * @mm: mm from which memcg should be extracted. It can be NULL. 862 * 863 * Obtain a reference on mm->memcg and returns it if successful. If mm 864 * is NULL, then the memcg is chosen as follows: 865 * 1) The active memcg, if set. 866 * 2) current->mm->memcg, if available 867 * 3) root memcg 868 * If mem_cgroup is disabled, NULL is returned. 869 */ 870 struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm) 871 { 872 struct mem_cgroup *memcg; 873 874 if (mem_cgroup_disabled()) 875 return NULL; 876 877 /* 878 * Page cache insertions can happen without an 879 * actual mm context, e.g. during disk probing 880 * on boot, loopback IO, acct() writes etc. 881 * 882 * No need to css_get on root memcg as the reference 883 * counting is disabled on the root level in the 884 * cgroup core. See CSS_NO_REF. 885 */ 886 if (unlikely(!mm)) { 887 memcg = active_memcg(); 888 if (unlikely(memcg)) { 889 /* remote memcg must hold a ref */ 890 css_get(&memcg->css); 891 return memcg; 892 } 893 mm = current->mm; 894 if (unlikely(!mm)) 895 return root_mem_cgroup; 896 } 897 898 rcu_read_lock(); 899 do { 900 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); 901 if (unlikely(!memcg)) 902 memcg = root_mem_cgroup; 903 } while (!css_tryget(&memcg->css)); 904 rcu_read_unlock(); 905 return memcg; 906 } 907 EXPORT_SYMBOL(get_mem_cgroup_from_mm); 908 909 static __always_inline bool memcg_kmem_bypass(void) 910 { 911 /* Allow remote memcg charging from any context. */ 912 if (unlikely(active_memcg())) 913 return false; 914 915 /* Memcg to charge can't be determined. */ 916 if (!in_task() || !current->mm || (current->flags & PF_KTHREAD)) 917 return true; 918 919 return false; 920 } 921 922 /** 923 * mem_cgroup_iter - iterate over memory cgroup hierarchy 924 * @root: hierarchy root 925 * @prev: previously returned memcg, NULL on first invocation 926 * @reclaim: cookie for shared reclaim walks, NULL for full walks 927 * 928 * Returns references to children of the hierarchy below @root, or 929 * @root itself, or %NULL after a full round-trip. 930 * 931 * Caller must pass the return value in @prev on subsequent 932 * invocations for reference counting, or use mem_cgroup_iter_break() 933 * to cancel a hierarchy walk before the round-trip is complete. 934 * 935 * Reclaimers can specify a node in @reclaim to divide up the memcgs 936 * in the hierarchy among all concurrent reclaimers operating on the 937 * same node. 938 */ 939 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root, 940 struct mem_cgroup *prev, 941 struct mem_cgroup_reclaim_cookie *reclaim) 942 { 943 struct mem_cgroup_reclaim_iter *iter; 944 struct cgroup_subsys_state *css = NULL; 945 struct mem_cgroup *memcg = NULL; 946 struct mem_cgroup *pos = NULL; 947 948 if (mem_cgroup_disabled()) 949 return NULL; 950 951 if (!root) 952 root = root_mem_cgroup; 953 954 if (prev && !reclaim) 955 pos = prev; 956 957 rcu_read_lock(); 958 959 if (reclaim) { 960 struct mem_cgroup_per_node *mz; 961 962 mz = root->nodeinfo[reclaim->pgdat->node_id]; 963 iter = &mz->iter; 964 965 if (prev && reclaim->generation != iter->generation) 966 goto out_unlock; 967 968 while (1) { 969 pos = READ_ONCE(iter->position); 970 if (!pos || css_tryget(&pos->css)) 971 break; 972 /* 973 * css reference reached zero, so iter->position will 974 * be cleared by ->css_released. However, we should not 975 * rely on this happening soon, because ->css_released 976 * is called from a work queue, and by busy-waiting we 977 * might block it. So we clear iter->position right 978 * away. 979 */ 980 (void)cmpxchg(&iter->position, pos, NULL); 981 } 982 } 983 984 if (pos) 985 css = &pos->css; 986 987 for (;;) { 988 css = css_next_descendant_pre(css, &root->css); 989 if (!css) { 990 /* 991 * Reclaimers share the hierarchy walk, and a 992 * new one might jump in right at the end of 993 * the hierarchy - make sure they see at least 994 * one group and restart from the beginning. 995 */ 996 if (!prev) 997 continue; 998 break; 999 } 1000 1001 /* 1002 * Verify the css and acquire a reference. The root 1003 * is provided by the caller, so we know it's alive 1004 * and kicking, and don't take an extra reference. 1005 */ 1006 memcg = mem_cgroup_from_css(css); 1007 1008 if (css == &root->css) 1009 break; 1010 1011 if (css_tryget(css)) 1012 break; 1013 1014 memcg = NULL; 1015 } 1016 1017 if (reclaim) { 1018 /* 1019 * The position could have already been updated by a competing 1020 * thread, so check that the value hasn't changed since we read 1021 * it to avoid reclaiming from the same cgroup twice. 1022 */ 1023 (void)cmpxchg(&iter->position, pos, memcg); 1024 1025 if (pos) 1026 css_put(&pos->css); 1027 1028 if (!memcg) 1029 iter->generation++; 1030 else if (!prev) 1031 reclaim->generation = iter->generation; 1032 } 1033 1034 out_unlock: 1035 rcu_read_unlock(); 1036 if (prev && prev != root) 1037 css_put(&prev->css); 1038 1039 return memcg; 1040 } 1041 1042 /** 1043 * mem_cgroup_iter_break - abort a hierarchy walk prematurely 1044 * @root: hierarchy root 1045 * @prev: last visited hierarchy member as returned by mem_cgroup_iter() 1046 */ 1047 void mem_cgroup_iter_break(struct mem_cgroup *root, 1048 struct mem_cgroup *prev) 1049 { 1050 if (!root) 1051 root = root_mem_cgroup; 1052 if (prev && prev != root) 1053 css_put(&prev->css); 1054 } 1055 1056 static void __invalidate_reclaim_iterators(struct mem_cgroup *from, 1057 struct mem_cgroup *dead_memcg) 1058 { 1059 struct mem_cgroup_reclaim_iter *iter; 1060 struct mem_cgroup_per_node *mz; 1061 int nid; 1062 1063 for_each_node(nid) { 1064 mz = from->nodeinfo[nid]; 1065 iter = &mz->iter; 1066 cmpxchg(&iter->position, dead_memcg, NULL); 1067 } 1068 } 1069 1070 static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg) 1071 { 1072 struct mem_cgroup *memcg = dead_memcg; 1073 struct mem_cgroup *last; 1074 1075 do { 1076 __invalidate_reclaim_iterators(memcg, dead_memcg); 1077 last = memcg; 1078 } while ((memcg = parent_mem_cgroup(memcg))); 1079 1080 /* 1081 * When cgruop1 non-hierarchy mode is used, 1082 * parent_mem_cgroup() does not walk all the way up to the 1083 * cgroup root (root_mem_cgroup). So we have to handle 1084 * dead_memcg from cgroup root separately. 1085 */ 1086 if (last != root_mem_cgroup) 1087 __invalidate_reclaim_iterators(root_mem_cgroup, 1088 dead_memcg); 1089 } 1090 1091 /** 1092 * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy 1093 * @memcg: hierarchy root 1094 * @fn: function to call for each task 1095 * @arg: argument passed to @fn 1096 * 1097 * This function iterates over tasks attached to @memcg or to any of its 1098 * descendants and calls @fn for each task. If @fn returns a non-zero 1099 * value, the function breaks the iteration loop and returns the value. 1100 * Otherwise, it will iterate over all tasks and return 0. 1101 * 1102 * This function must not be called for the root memory cgroup. 1103 */ 1104 int mem_cgroup_scan_tasks(struct mem_cgroup *memcg, 1105 int (*fn)(struct task_struct *, void *), void *arg) 1106 { 1107 struct mem_cgroup *iter; 1108 int ret = 0; 1109 1110 BUG_ON(memcg == root_mem_cgroup); 1111 1112 for_each_mem_cgroup_tree(iter, memcg) { 1113 struct css_task_iter it; 1114 struct task_struct *task; 1115 1116 css_task_iter_start(&iter->css, CSS_TASK_ITER_PROCS, &it); 1117 while (!ret && (task = css_task_iter_next(&it))) 1118 ret = fn(task, arg); 1119 css_task_iter_end(&it); 1120 if (ret) { 1121 mem_cgroup_iter_break(memcg, iter); 1122 break; 1123 } 1124 } 1125 return ret; 1126 } 1127 1128 #ifdef CONFIG_DEBUG_VM 1129 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio) 1130 { 1131 struct mem_cgroup *memcg; 1132 1133 if (mem_cgroup_disabled()) 1134 return; 1135 1136 memcg = folio_memcg(folio); 1137 1138 if (!memcg) 1139 VM_BUG_ON_FOLIO(lruvec_memcg(lruvec) != root_mem_cgroup, folio); 1140 else 1141 VM_BUG_ON_FOLIO(lruvec_memcg(lruvec) != memcg, folio); 1142 } 1143 #endif 1144 1145 /** 1146 * folio_lruvec_lock - Lock the lruvec for a folio. 1147 * @folio: Pointer to the folio. 1148 * 1149 * These functions are safe to use under any of the following conditions: 1150 * - folio locked 1151 * - folio_test_lru false 1152 * - folio_memcg_lock() 1153 * - folio frozen (refcount of 0) 1154 * 1155 * Return: The lruvec this folio is on with its lock held. 1156 */ 1157 struct lruvec *folio_lruvec_lock(struct folio *folio) 1158 { 1159 struct lruvec *lruvec = folio_lruvec(folio); 1160 1161 spin_lock(&lruvec->lru_lock); 1162 lruvec_memcg_debug(lruvec, folio); 1163 1164 return lruvec; 1165 } 1166 1167 /** 1168 * folio_lruvec_lock_irq - Lock the lruvec for a folio. 1169 * @folio: Pointer to the folio. 1170 * 1171 * These functions are safe to use under any of the following conditions: 1172 * - folio locked 1173 * - folio_test_lru false 1174 * - folio_memcg_lock() 1175 * - folio frozen (refcount of 0) 1176 * 1177 * Return: The lruvec this folio is on with its lock held and interrupts 1178 * disabled. 1179 */ 1180 struct lruvec *folio_lruvec_lock_irq(struct folio *folio) 1181 { 1182 struct lruvec *lruvec = folio_lruvec(folio); 1183 1184 spin_lock_irq(&lruvec->lru_lock); 1185 lruvec_memcg_debug(lruvec, folio); 1186 1187 return lruvec; 1188 } 1189 1190 /** 1191 * folio_lruvec_lock_irqsave - Lock the lruvec for a folio. 1192 * @folio: Pointer to the folio. 1193 * @flags: Pointer to irqsave flags. 1194 * 1195 * These functions are safe to use under any of the following conditions: 1196 * - folio locked 1197 * - folio_test_lru false 1198 * - folio_memcg_lock() 1199 * - folio frozen (refcount of 0) 1200 * 1201 * Return: The lruvec this folio is on with its lock held and interrupts 1202 * disabled. 1203 */ 1204 struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio, 1205 unsigned long *flags) 1206 { 1207 struct lruvec *lruvec = folio_lruvec(folio); 1208 1209 spin_lock_irqsave(&lruvec->lru_lock, *flags); 1210 lruvec_memcg_debug(lruvec, folio); 1211 1212 return lruvec; 1213 } 1214 1215 /** 1216 * mem_cgroup_update_lru_size - account for adding or removing an lru page 1217 * @lruvec: mem_cgroup per zone lru vector 1218 * @lru: index of lru list the page is sitting on 1219 * @zid: zone id of the accounted pages 1220 * @nr_pages: positive when adding or negative when removing 1221 * 1222 * This function must be called under lru_lock, just before a page is added 1223 * to or just after a page is removed from an lru list (that ordering being 1224 * so as to allow it to check that lru_size 0 is consistent with list_empty). 1225 */ 1226 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, 1227 int zid, int nr_pages) 1228 { 1229 struct mem_cgroup_per_node *mz; 1230 unsigned long *lru_size; 1231 long size; 1232 1233 if (mem_cgroup_disabled()) 1234 return; 1235 1236 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); 1237 lru_size = &mz->lru_zone_size[zid][lru]; 1238 1239 if (nr_pages < 0) 1240 *lru_size += nr_pages; 1241 1242 size = *lru_size; 1243 if (WARN_ONCE(size < 0, 1244 "%s(%p, %d, %d): lru_size %ld\n", 1245 __func__, lruvec, lru, nr_pages, size)) { 1246 VM_BUG_ON(1); 1247 *lru_size = 0; 1248 } 1249 1250 if (nr_pages > 0) 1251 *lru_size += nr_pages; 1252 } 1253 1254 /** 1255 * mem_cgroup_margin - calculate chargeable space of a memory cgroup 1256 * @memcg: the memory cgroup 1257 * 1258 * Returns the maximum amount of memory @mem can be charged with, in 1259 * pages. 1260 */ 1261 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg) 1262 { 1263 unsigned long margin = 0; 1264 unsigned long count; 1265 unsigned long limit; 1266 1267 count = page_counter_read(&memcg->memory); 1268 limit = READ_ONCE(memcg->memory.max); 1269 if (count < limit) 1270 margin = limit - count; 1271 1272 if (do_memsw_account()) { 1273 count = page_counter_read(&memcg->memsw); 1274 limit = READ_ONCE(memcg->memsw.max); 1275 if (count < limit) 1276 margin = min(margin, limit - count); 1277 else 1278 margin = 0; 1279 } 1280 1281 return margin; 1282 } 1283 1284 /* 1285 * A routine for checking "mem" is under move_account() or not. 1286 * 1287 * Checking a cgroup is mc.from or mc.to or under hierarchy of 1288 * moving cgroups. This is for waiting at high-memory pressure 1289 * caused by "move". 1290 */ 1291 static bool mem_cgroup_under_move(struct mem_cgroup *memcg) 1292 { 1293 struct mem_cgroup *from; 1294 struct mem_cgroup *to; 1295 bool ret = false; 1296 /* 1297 * Unlike task_move routines, we access mc.to, mc.from not under 1298 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead. 1299 */ 1300 spin_lock(&mc.lock); 1301 from = mc.from; 1302 to = mc.to; 1303 if (!from) 1304 goto unlock; 1305 1306 ret = mem_cgroup_is_descendant(from, memcg) || 1307 mem_cgroup_is_descendant(to, memcg); 1308 unlock: 1309 spin_unlock(&mc.lock); 1310 return ret; 1311 } 1312 1313 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg) 1314 { 1315 if (mc.moving_task && current != mc.moving_task) { 1316 if (mem_cgroup_under_move(memcg)) { 1317 DEFINE_WAIT(wait); 1318 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE); 1319 /* moving charge context might have finished. */ 1320 if (mc.moving_task) 1321 schedule(); 1322 finish_wait(&mc.waitq, &wait); 1323 return true; 1324 } 1325 } 1326 return false; 1327 } 1328 1329 struct memory_stat { 1330 const char *name; 1331 unsigned int idx; 1332 }; 1333 1334 static const struct memory_stat memory_stats[] = { 1335 { "anon", NR_ANON_MAPPED }, 1336 { "file", NR_FILE_PAGES }, 1337 { "kernel_stack", NR_KERNEL_STACK_KB }, 1338 { "pagetables", NR_PAGETABLE }, 1339 { "percpu", MEMCG_PERCPU_B }, 1340 { "sock", MEMCG_SOCK }, 1341 { "shmem", NR_SHMEM }, 1342 { "file_mapped", NR_FILE_MAPPED }, 1343 { "file_dirty", NR_FILE_DIRTY }, 1344 { "file_writeback", NR_WRITEBACK }, 1345 #ifdef CONFIG_SWAP 1346 { "swapcached", NR_SWAPCACHE }, 1347 #endif 1348 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1349 { "anon_thp", NR_ANON_THPS }, 1350 { "file_thp", NR_FILE_THPS }, 1351 { "shmem_thp", NR_SHMEM_THPS }, 1352 #endif 1353 { "inactive_anon", NR_INACTIVE_ANON }, 1354 { "active_anon", NR_ACTIVE_ANON }, 1355 { "inactive_file", NR_INACTIVE_FILE }, 1356 { "active_file", NR_ACTIVE_FILE }, 1357 { "unevictable", NR_UNEVICTABLE }, 1358 { "slab_reclaimable", NR_SLAB_RECLAIMABLE_B }, 1359 { "slab_unreclaimable", NR_SLAB_UNRECLAIMABLE_B }, 1360 1361 /* The memory events */ 1362 { "workingset_refault_anon", WORKINGSET_REFAULT_ANON }, 1363 { "workingset_refault_file", WORKINGSET_REFAULT_FILE }, 1364 { "workingset_activate_anon", WORKINGSET_ACTIVATE_ANON }, 1365 { "workingset_activate_file", WORKINGSET_ACTIVATE_FILE }, 1366 { "workingset_restore_anon", WORKINGSET_RESTORE_ANON }, 1367 { "workingset_restore_file", WORKINGSET_RESTORE_FILE }, 1368 { "workingset_nodereclaim", WORKINGSET_NODERECLAIM }, 1369 }; 1370 1371 /* Translate stat items to the correct unit for memory.stat output */ 1372 static int memcg_page_state_unit(int item) 1373 { 1374 switch (item) { 1375 case MEMCG_PERCPU_B: 1376 case NR_SLAB_RECLAIMABLE_B: 1377 case NR_SLAB_UNRECLAIMABLE_B: 1378 case WORKINGSET_REFAULT_ANON: 1379 case WORKINGSET_REFAULT_FILE: 1380 case WORKINGSET_ACTIVATE_ANON: 1381 case WORKINGSET_ACTIVATE_FILE: 1382 case WORKINGSET_RESTORE_ANON: 1383 case WORKINGSET_RESTORE_FILE: 1384 case WORKINGSET_NODERECLAIM: 1385 return 1; 1386 case NR_KERNEL_STACK_KB: 1387 return SZ_1K; 1388 default: 1389 return PAGE_SIZE; 1390 } 1391 } 1392 1393 static inline unsigned long memcg_page_state_output(struct mem_cgroup *memcg, 1394 int item) 1395 { 1396 return memcg_page_state(memcg, item) * memcg_page_state_unit(item); 1397 } 1398 1399 static char *memory_stat_format(struct mem_cgroup *memcg) 1400 { 1401 struct seq_buf s; 1402 int i; 1403 1404 seq_buf_init(&s, kmalloc(PAGE_SIZE, GFP_KERNEL), PAGE_SIZE); 1405 if (!s.buffer) 1406 return NULL; 1407 1408 /* 1409 * Provide statistics on the state of the memory subsystem as 1410 * well as cumulative event counters that show past behavior. 1411 * 1412 * This list is ordered following a combination of these gradients: 1413 * 1) generic big picture -> specifics and details 1414 * 2) reflecting userspace activity -> reflecting kernel heuristics 1415 * 1416 * Current memory state: 1417 */ 1418 cgroup_rstat_flush(memcg->css.cgroup); 1419 1420 for (i = 0; i < ARRAY_SIZE(memory_stats); i++) { 1421 u64 size; 1422 1423 size = memcg_page_state_output(memcg, memory_stats[i].idx); 1424 seq_buf_printf(&s, "%s %llu\n", memory_stats[i].name, size); 1425 1426 if (unlikely(memory_stats[i].idx == NR_SLAB_UNRECLAIMABLE_B)) { 1427 size += memcg_page_state_output(memcg, 1428 NR_SLAB_RECLAIMABLE_B); 1429 seq_buf_printf(&s, "slab %llu\n", size); 1430 } 1431 } 1432 1433 /* Accumulated memory events */ 1434 1435 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGFAULT), 1436 memcg_events(memcg, PGFAULT)); 1437 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGMAJFAULT), 1438 memcg_events(memcg, PGMAJFAULT)); 1439 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGREFILL), 1440 memcg_events(memcg, PGREFILL)); 1441 seq_buf_printf(&s, "pgscan %lu\n", 1442 memcg_events(memcg, PGSCAN_KSWAPD) + 1443 memcg_events(memcg, PGSCAN_DIRECT)); 1444 seq_buf_printf(&s, "pgsteal %lu\n", 1445 memcg_events(memcg, PGSTEAL_KSWAPD) + 1446 memcg_events(memcg, PGSTEAL_DIRECT)); 1447 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGACTIVATE), 1448 memcg_events(memcg, PGACTIVATE)); 1449 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGDEACTIVATE), 1450 memcg_events(memcg, PGDEACTIVATE)); 1451 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGLAZYFREE), 1452 memcg_events(memcg, PGLAZYFREE)); 1453 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGLAZYFREED), 1454 memcg_events(memcg, PGLAZYFREED)); 1455 1456 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1457 seq_buf_printf(&s, "%s %lu\n", vm_event_name(THP_FAULT_ALLOC), 1458 memcg_events(memcg, THP_FAULT_ALLOC)); 1459 seq_buf_printf(&s, "%s %lu\n", vm_event_name(THP_COLLAPSE_ALLOC), 1460 memcg_events(memcg, THP_COLLAPSE_ALLOC)); 1461 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 1462 1463 /* The above should easily fit into one page */ 1464 WARN_ON_ONCE(seq_buf_has_overflowed(&s)); 1465 1466 return s.buffer; 1467 } 1468 1469 #define K(x) ((x) << (PAGE_SHIFT-10)) 1470 /** 1471 * mem_cgroup_print_oom_context: Print OOM information relevant to 1472 * memory controller. 1473 * @memcg: The memory cgroup that went over limit 1474 * @p: Task that is going to be killed 1475 * 1476 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is 1477 * enabled 1478 */ 1479 void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p) 1480 { 1481 rcu_read_lock(); 1482 1483 if (memcg) { 1484 pr_cont(",oom_memcg="); 1485 pr_cont_cgroup_path(memcg->css.cgroup); 1486 } else 1487 pr_cont(",global_oom"); 1488 if (p) { 1489 pr_cont(",task_memcg="); 1490 pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id)); 1491 } 1492 rcu_read_unlock(); 1493 } 1494 1495 /** 1496 * mem_cgroup_print_oom_meminfo: Print OOM memory information relevant to 1497 * memory controller. 1498 * @memcg: The memory cgroup that went over limit 1499 */ 1500 void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg) 1501 { 1502 char *buf; 1503 1504 pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n", 1505 K((u64)page_counter_read(&memcg->memory)), 1506 K((u64)READ_ONCE(memcg->memory.max)), memcg->memory.failcnt); 1507 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 1508 pr_info("swap: usage %llukB, limit %llukB, failcnt %lu\n", 1509 K((u64)page_counter_read(&memcg->swap)), 1510 K((u64)READ_ONCE(memcg->swap.max)), memcg->swap.failcnt); 1511 else { 1512 pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n", 1513 K((u64)page_counter_read(&memcg->memsw)), 1514 K((u64)memcg->memsw.max), memcg->memsw.failcnt); 1515 pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n", 1516 K((u64)page_counter_read(&memcg->kmem)), 1517 K((u64)memcg->kmem.max), memcg->kmem.failcnt); 1518 } 1519 1520 pr_info("Memory cgroup stats for "); 1521 pr_cont_cgroup_path(memcg->css.cgroup); 1522 pr_cont(":"); 1523 buf = memory_stat_format(memcg); 1524 if (!buf) 1525 return; 1526 pr_info("%s", buf); 1527 kfree(buf); 1528 } 1529 1530 /* 1531 * Return the memory (and swap, if configured) limit for a memcg. 1532 */ 1533 unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg) 1534 { 1535 unsigned long max = READ_ONCE(memcg->memory.max); 1536 1537 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) { 1538 if (mem_cgroup_swappiness(memcg)) 1539 max += min(READ_ONCE(memcg->swap.max), 1540 (unsigned long)total_swap_pages); 1541 } else { /* v1 */ 1542 if (mem_cgroup_swappiness(memcg)) { 1543 /* Calculate swap excess capacity from memsw limit */ 1544 unsigned long swap = READ_ONCE(memcg->memsw.max) - max; 1545 1546 max += min(swap, (unsigned long)total_swap_pages); 1547 } 1548 } 1549 return max; 1550 } 1551 1552 unsigned long mem_cgroup_size(struct mem_cgroup *memcg) 1553 { 1554 return page_counter_read(&memcg->memory); 1555 } 1556 1557 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask, 1558 int order) 1559 { 1560 struct oom_control oc = { 1561 .zonelist = NULL, 1562 .nodemask = NULL, 1563 .memcg = memcg, 1564 .gfp_mask = gfp_mask, 1565 .order = order, 1566 }; 1567 bool ret = true; 1568 1569 if (mutex_lock_killable(&oom_lock)) 1570 return true; 1571 1572 if (mem_cgroup_margin(memcg) >= (1 << order)) 1573 goto unlock; 1574 1575 /* 1576 * A few threads which were not waiting at mutex_lock_killable() can 1577 * fail to bail out. Therefore, check again after holding oom_lock. 1578 */ 1579 ret = should_force_charge() || out_of_memory(&oc); 1580 1581 unlock: 1582 mutex_unlock(&oom_lock); 1583 return ret; 1584 } 1585 1586 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg, 1587 pg_data_t *pgdat, 1588 gfp_t gfp_mask, 1589 unsigned long *total_scanned) 1590 { 1591 struct mem_cgroup *victim = NULL; 1592 int total = 0; 1593 int loop = 0; 1594 unsigned long excess; 1595 unsigned long nr_scanned; 1596 struct mem_cgroup_reclaim_cookie reclaim = { 1597 .pgdat = pgdat, 1598 }; 1599 1600 excess = soft_limit_excess(root_memcg); 1601 1602 while (1) { 1603 victim = mem_cgroup_iter(root_memcg, victim, &reclaim); 1604 if (!victim) { 1605 loop++; 1606 if (loop >= 2) { 1607 /* 1608 * If we have not been able to reclaim 1609 * anything, it might because there are 1610 * no reclaimable pages under this hierarchy 1611 */ 1612 if (!total) 1613 break; 1614 /* 1615 * We want to do more targeted reclaim. 1616 * excess >> 2 is not to excessive so as to 1617 * reclaim too much, nor too less that we keep 1618 * coming back to reclaim from this cgroup 1619 */ 1620 if (total >= (excess >> 2) || 1621 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) 1622 break; 1623 } 1624 continue; 1625 } 1626 total += mem_cgroup_shrink_node(victim, gfp_mask, false, 1627 pgdat, &nr_scanned); 1628 *total_scanned += nr_scanned; 1629 if (!soft_limit_excess(root_memcg)) 1630 break; 1631 } 1632 mem_cgroup_iter_break(root_memcg, victim); 1633 return total; 1634 } 1635 1636 #ifdef CONFIG_LOCKDEP 1637 static struct lockdep_map memcg_oom_lock_dep_map = { 1638 .name = "memcg_oom_lock", 1639 }; 1640 #endif 1641 1642 static DEFINE_SPINLOCK(memcg_oom_lock); 1643 1644 /* 1645 * Check OOM-Killer is already running under our hierarchy. 1646 * If someone is running, return false. 1647 */ 1648 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg) 1649 { 1650 struct mem_cgroup *iter, *failed = NULL; 1651 1652 spin_lock(&memcg_oom_lock); 1653 1654 for_each_mem_cgroup_tree(iter, memcg) { 1655 if (iter->oom_lock) { 1656 /* 1657 * this subtree of our hierarchy is already locked 1658 * so we cannot give a lock. 1659 */ 1660 failed = iter; 1661 mem_cgroup_iter_break(memcg, iter); 1662 break; 1663 } else 1664 iter->oom_lock = true; 1665 } 1666 1667 if (failed) { 1668 /* 1669 * OK, we failed to lock the whole subtree so we have 1670 * to clean up what we set up to the failing subtree 1671 */ 1672 for_each_mem_cgroup_tree(iter, memcg) { 1673 if (iter == failed) { 1674 mem_cgroup_iter_break(memcg, iter); 1675 break; 1676 } 1677 iter->oom_lock = false; 1678 } 1679 } else 1680 mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_); 1681 1682 spin_unlock(&memcg_oom_lock); 1683 1684 return !failed; 1685 } 1686 1687 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg) 1688 { 1689 struct mem_cgroup *iter; 1690 1691 spin_lock(&memcg_oom_lock); 1692 mutex_release(&memcg_oom_lock_dep_map, _RET_IP_); 1693 for_each_mem_cgroup_tree(iter, memcg) 1694 iter->oom_lock = false; 1695 spin_unlock(&memcg_oom_lock); 1696 } 1697 1698 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg) 1699 { 1700 struct mem_cgroup *iter; 1701 1702 spin_lock(&memcg_oom_lock); 1703 for_each_mem_cgroup_tree(iter, memcg) 1704 iter->under_oom++; 1705 spin_unlock(&memcg_oom_lock); 1706 } 1707 1708 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg) 1709 { 1710 struct mem_cgroup *iter; 1711 1712 /* 1713 * Be careful about under_oom underflows because a child memcg 1714 * could have been added after mem_cgroup_mark_under_oom. 1715 */ 1716 spin_lock(&memcg_oom_lock); 1717 for_each_mem_cgroup_tree(iter, memcg) 1718 if (iter->under_oom > 0) 1719 iter->under_oom--; 1720 spin_unlock(&memcg_oom_lock); 1721 } 1722 1723 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq); 1724 1725 struct oom_wait_info { 1726 struct mem_cgroup *memcg; 1727 wait_queue_entry_t wait; 1728 }; 1729 1730 static int memcg_oom_wake_function(wait_queue_entry_t *wait, 1731 unsigned mode, int sync, void *arg) 1732 { 1733 struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg; 1734 struct mem_cgroup *oom_wait_memcg; 1735 struct oom_wait_info *oom_wait_info; 1736 1737 oom_wait_info = container_of(wait, struct oom_wait_info, wait); 1738 oom_wait_memcg = oom_wait_info->memcg; 1739 1740 if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) && 1741 !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg)) 1742 return 0; 1743 return autoremove_wake_function(wait, mode, sync, arg); 1744 } 1745 1746 static void memcg_oom_recover(struct mem_cgroup *memcg) 1747 { 1748 /* 1749 * For the following lockless ->under_oom test, the only required 1750 * guarantee is that it must see the state asserted by an OOM when 1751 * this function is called as a result of userland actions 1752 * triggered by the notification of the OOM. This is trivially 1753 * achieved by invoking mem_cgroup_mark_under_oom() before 1754 * triggering notification. 1755 */ 1756 if (memcg && memcg->under_oom) 1757 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg); 1758 } 1759 1760 enum oom_status { 1761 OOM_SUCCESS, 1762 OOM_FAILED, 1763 OOM_ASYNC, 1764 OOM_SKIPPED 1765 }; 1766 1767 static enum oom_status mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order) 1768 { 1769 enum oom_status ret; 1770 bool locked; 1771 1772 if (order > PAGE_ALLOC_COSTLY_ORDER) 1773 return OOM_SKIPPED; 1774 1775 memcg_memory_event(memcg, MEMCG_OOM); 1776 1777 /* 1778 * We are in the middle of the charge context here, so we 1779 * don't want to block when potentially sitting on a callstack 1780 * that holds all kinds of filesystem and mm locks. 1781 * 1782 * cgroup1 allows disabling the OOM killer and waiting for outside 1783 * handling until the charge can succeed; remember the context and put 1784 * the task to sleep at the end of the page fault when all locks are 1785 * released. 1786 * 1787 * On the other hand, in-kernel OOM killer allows for an async victim 1788 * memory reclaim (oom_reaper) and that means that we are not solely 1789 * relying on the oom victim to make a forward progress and we can 1790 * invoke the oom killer here. 1791 * 1792 * Please note that mem_cgroup_out_of_memory might fail to find a 1793 * victim and then we have to bail out from the charge path. 1794 */ 1795 if (memcg->oom_kill_disable) { 1796 if (!current->in_user_fault) 1797 return OOM_SKIPPED; 1798 css_get(&memcg->css); 1799 current->memcg_in_oom = memcg; 1800 current->memcg_oom_gfp_mask = mask; 1801 current->memcg_oom_order = order; 1802 1803 return OOM_ASYNC; 1804 } 1805 1806 mem_cgroup_mark_under_oom(memcg); 1807 1808 locked = mem_cgroup_oom_trylock(memcg); 1809 1810 if (locked) 1811 mem_cgroup_oom_notify(memcg); 1812 1813 mem_cgroup_unmark_under_oom(memcg); 1814 if (mem_cgroup_out_of_memory(memcg, mask, order)) 1815 ret = OOM_SUCCESS; 1816 else 1817 ret = OOM_FAILED; 1818 1819 if (locked) 1820 mem_cgroup_oom_unlock(memcg); 1821 1822 return ret; 1823 } 1824 1825 /** 1826 * mem_cgroup_oom_synchronize - complete memcg OOM handling 1827 * @handle: actually kill/wait or just clean up the OOM state 1828 * 1829 * This has to be called at the end of a page fault if the memcg OOM 1830 * handler was enabled. 1831 * 1832 * Memcg supports userspace OOM handling where failed allocations must 1833 * sleep on a waitqueue until the userspace task resolves the 1834 * situation. Sleeping directly in the charge context with all kinds 1835 * of locks held is not a good idea, instead we remember an OOM state 1836 * in the task and mem_cgroup_oom_synchronize() has to be called at 1837 * the end of the page fault to complete the OOM handling. 1838 * 1839 * Returns %true if an ongoing memcg OOM situation was detected and 1840 * completed, %false otherwise. 1841 */ 1842 bool mem_cgroup_oom_synchronize(bool handle) 1843 { 1844 struct mem_cgroup *memcg = current->memcg_in_oom; 1845 struct oom_wait_info owait; 1846 bool locked; 1847 1848 /* OOM is global, do not handle */ 1849 if (!memcg) 1850 return false; 1851 1852 if (!handle) 1853 goto cleanup; 1854 1855 owait.memcg = memcg; 1856 owait.wait.flags = 0; 1857 owait.wait.func = memcg_oom_wake_function; 1858 owait.wait.private = current; 1859 INIT_LIST_HEAD(&owait.wait.entry); 1860 1861 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE); 1862 mem_cgroup_mark_under_oom(memcg); 1863 1864 locked = mem_cgroup_oom_trylock(memcg); 1865 1866 if (locked) 1867 mem_cgroup_oom_notify(memcg); 1868 1869 if (locked && !memcg->oom_kill_disable) { 1870 mem_cgroup_unmark_under_oom(memcg); 1871 finish_wait(&memcg_oom_waitq, &owait.wait); 1872 mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask, 1873 current->memcg_oom_order); 1874 } else { 1875 schedule(); 1876 mem_cgroup_unmark_under_oom(memcg); 1877 finish_wait(&memcg_oom_waitq, &owait.wait); 1878 } 1879 1880 if (locked) { 1881 mem_cgroup_oom_unlock(memcg); 1882 /* 1883 * There is no guarantee that an OOM-lock contender 1884 * sees the wakeups triggered by the OOM kill 1885 * uncharges. Wake any sleepers explicitly. 1886 */ 1887 memcg_oom_recover(memcg); 1888 } 1889 cleanup: 1890 current->memcg_in_oom = NULL; 1891 css_put(&memcg->css); 1892 return true; 1893 } 1894 1895 /** 1896 * mem_cgroup_get_oom_group - get a memory cgroup to clean up after OOM 1897 * @victim: task to be killed by the OOM killer 1898 * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM 1899 * 1900 * Returns a pointer to a memory cgroup, which has to be cleaned up 1901 * by killing all belonging OOM-killable tasks. 1902 * 1903 * Caller has to call mem_cgroup_put() on the returned non-NULL memcg. 1904 */ 1905 struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim, 1906 struct mem_cgroup *oom_domain) 1907 { 1908 struct mem_cgroup *oom_group = NULL; 1909 struct mem_cgroup *memcg; 1910 1911 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) 1912 return NULL; 1913 1914 if (!oom_domain) 1915 oom_domain = root_mem_cgroup; 1916 1917 rcu_read_lock(); 1918 1919 memcg = mem_cgroup_from_task(victim); 1920 if (memcg == root_mem_cgroup) 1921 goto out; 1922 1923 /* 1924 * If the victim task has been asynchronously moved to a different 1925 * memory cgroup, we might end up killing tasks outside oom_domain. 1926 * In this case it's better to ignore memory.group.oom. 1927 */ 1928 if (unlikely(!mem_cgroup_is_descendant(memcg, oom_domain))) 1929 goto out; 1930 1931 /* 1932 * Traverse the memory cgroup hierarchy from the victim task's 1933 * cgroup up to the OOMing cgroup (or root) to find the 1934 * highest-level memory cgroup with oom.group set. 1935 */ 1936 for (; memcg; memcg = parent_mem_cgroup(memcg)) { 1937 if (memcg->oom_group) 1938 oom_group = memcg; 1939 1940 if (memcg == oom_domain) 1941 break; 1942 } 1943 1944 if (oom_group) 1945 css_get(&oom_group->css); 1946 out: 1947 rcu_read_unlock(); 1948 1949 return oom_group; 1950 } 1951 1952 void mem_cgroup_print_oom_group(struct mem_cgroup *memcg) 1953 { 1954 pr_info("Tasks in "); 1955 pr_cont_cgroup_path(memcg->css.cgroup); 1956 pr_cont(" are going to be killed due to memory.oom.group set\n"); 1957 } 1958 1959 /** 1960 * folio_memcg_lock - Bind a folio to its memcg. 1961 * @folio: The folio. 1962 * 1963 * This function prevents unlocked LRU folios from being moved to 1964 * another cgroup. 1965 * 1966 * It ensures lifetime of the bound memcg. The caller is responsible 1967 * for the lifetime of the folio. 1968 */ 1969 void folio_memcg_lock(struct folio *folio) 1970 { 1971 struct mem_cgroup *memcg; 1972 unsigned long flags; 1973 1974 /* 1975 * The RCU lock is held throughout the transaction. The fast 1976 * path can get away without acquiring the memcg->move_lock 1977 * because page moving starts with an RCU grace period. 1978 */ 1979 rcu_read_lock(); 1980 1981 if (mem_cgroup_disabled()) 1982 return; 1983 again: 1984 memcg = folio_memcg(folio); 1985 if (unlikely(!memcg)) 1986 return; 1987 1988 #ifdef CONFIG_PROVE_LOCKING 1989 local_irq_save(flags); 1990 might_lock(&memcg->move_lock); 1991 local_irq_restore(flags); 1992 #endif 1993 1994 if (atomic_read(&memcg->moving_account) <= 0) 1995 return; 1996 1997 spin_lock_irqsave(&memcg->move_lock, flags); 1998 if (memcg != folio_memcg(folio)) { 1999 spin_unlock_irqrestore(&memcg->move_lock, flags); 2000 goto again; 2001 } 2002 2003 /* 2004 * When charge migration first begins, we can have multiple 2005 * critical sections holding the fast-path RCU lock and one 2006 * holding the slowpath move_lock. Track the task who has the 2007 * move_lock for unlock_page_memcg(). 2008 */ 2009 memcg->move_lock_task = current; 2010 memcg->move_lock_flags = flags; 2011 } 2012 EXPORT_SYMBOL(folio_memcg_lock); 2013 2014 void lock_page_memcg(struct page *page) 2015 { 2016 folio_memcg_lock(page_folio(page)); 2017 } 2018 EXPORT_SYMBOL(lock_page_memcg); 2019 2020 static void __folio_memcg_unlock(struct mem_cgroup *memcg) 2021 { 2022 if (memcg && memcg->move_lock_task == current) { 2023 unsigned long flags = memcg->move_lock_flags; 2024 2025 memcg->move_lock_task = NULL; 2026 memcg->move_lock_flags = 0; 2027 2028 spin_unlock_irqrestore(&memcg->move_lock, flags); 2029 } 2030 2031 rcu_read_unlock(); 2032 } 2033 2034 /** 2035 * folio_memcg_unlock - Release the binding between a folio and its memcg. 2036 * @folio: The folio. 2037 * 2038 * This releases the binding created by folio_memcg_lock(). This does 2039 * not change the accounting of this folio to its memcg, but it does 2040 * permit others to change it. 2041 */ 2042 void folio_memcg_unlock(struct folio *folio) 2043 { 2044 __folio_memcg_unlock(folio_memcg(folio)); 2045 } 2046 EXPORT_SYMBOL(folio_memcg_unlock); 2047 2048 void unlock_page_memcg(struct page *page) 2049 { 2050 folio_memcg_unlock(page_folio(page)); 2051 } 2052 EXPORT_SYMBOL(unlock_page_memcg); 2053 2054 struct obj_stock { 2055 #ifdef CONFIG_MEMCG_KMEM 2056 struct obj_cgroup *cached_objcg; 2057 struct pglist_data *cached_pgdat; 2058 unsigned int nr_bytes; 2059 int nr_slab_reclaimable_b; 2060 int nr_slab_unreclaimable_b; 2061 #else 2062 int dummy[0]; 2063 #endif 2064 }; 2065 2066 struct memcg_stock_pcp { 2067 struct mem_cgroup *cached; /* this never be root cgroup */ 2068 unsigned int nr_pages; 2069 struct obj_stock task_obj; 2070 struct obj_stock irq_obj; 2071 2072 struct work_struct work; 2073 unsigned long flags; 2074 #define FLUSHING_CACHED_CHARGE 0 2075 }; 2076 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock); 2077 static DEFINE_MUTEX(percpu_charge_mutex); 2078 2079 #ifdef CONFIG_MEMCG_KMEM 2080 static void drain_obj_stock(struct obj_stock *stock); 2081 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock, 2082 struct mem_cgroup *root_memcg); 2083 2084 #else 2085 static inline void drain_obj_stock(struct obj_stock *stock) 2086 { 2087 } 2088 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock, 2089 struct mem_cgroup *root_memcg) 2090 { 2091 return false; 2092 } 2093 #endif 2094 2095 /* 2096 * Most kmem_cache_alloc() calls are from user context. The irq disable/enable 2097 * sequence used in this case to access content from object stock is slow. 2098 * To optimize for user context access, there are now two object stocks for 2099 * task context and interrupt context access respectively. 2100 * 2101 * The task context object stock can be accessed by disabling preemption only 2102 * which is cheap in non-preempt kernel. The interrupt context object stock 2103 * can only be accessed after disabling interrupt. User context code can 2104 * access interrupt object stock, but not vice versa. 2105 */ 2106 static inline struct obj_stock *get_obj_stock(unsigned long *pflags) 2107 { 2108 struct memcg_stock_pcp *stock; 2109 2110 if (likely(in_task())) { 2111 *pflags = 0UL; 2112 preempt_disable(); 2113 stock = this_cpu_ptr(&memcg_stock); 2114 return &stock->task_obj; 2115 } 2116 2117 local_irq_save(*pflags); 2118 stock = this_cpu_ptr(&memcg_stock); 2119 return &stock->irq_obj; 2120 } 2121 2122 static inline void put_obj_stock(unsigned long flags) 2123 { 2124 if (likely(in_task())) 2125 preempt_enable(); 2126 else 2127 local_irq_restore(flags); 2128 } 2129 2130 /** 2131 * consume_stock: Try to consume stocked charge on this cpu. 2132 * @memcg: memcg to consume from. 2133 * @nr_pages: how many pages to charge. 2134 * 2135 * The charges will only happen if @memcg matches the current cpu's memcg 2136 * stock, and at least @nr_pages are available in that stock. Failure to 2137 * service an allocation will refill the stock. 2138 * 2139 * returns true if successful, false otherwise. 2140 */ 2141 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) 2142 { 2143 struct memcg_stock_pcp *stock; 2144 unsigned long flags; 2145 bool ret = false; 2146 2147 if (nr_pages > MEMCG_CHARGE_BATCH) 2148 return ret; 2149 2150 local_irq_save(flags); 2151 2152 stock = this_cpu_ptr(&memcg_stock); 2153 if (memcg == stock->cached && stock->nr_pages >= nr_pages) { 2154 stock->nr_pages -= nr_pages; 2155 ret = true; 2156 } 2157 2158 local_irq_restore(flags); 2159 2160 return ret; 2161 } 2162 2163 /* 2164 * Returns stocks cached in percpu and reset cached information. 2165 */ 2166 static void drain_stock(struct memcg_stock_pcp *stock) 2167 { 2168 struct mem_cgroup *old = stock->cached; 2169 2170 if (!old) 2171 return; 2172 2173 if (stock->nr_pages) { 2174 page_counter_uncharge(&old->memory, stock->nr_pages); 2175 if (do_memsw_account()) 2176 page_counter_uncharge(&old->memsw, stock->nr_pages); 2177 stock->nr_pages = 0; 2178 } 2179 2180 css_put(&old->css); 2181 stock->cached = NULL; 2182 } 2183 2184 static void drain_local_stock(struct work_struct *dummy) 2185 { 2186 struct memcg_stock_pcp *stock; 2187 unsigned long flags; 2188 2189 /* 2190 * The only protection from cpu hotplug (memcg_hotplug_cpu_dead) vs. 2191 * drain_stock races is that we always operate on local CPU stock 2192 * here with IRQ disabled 2193 */ 2194 local_irq_save(flags); 2195 2196 stock = this_cpu_ptr(&memcg_stock); 2197 drain_obj_stock(&stock->irq_obj); 2198 if (in_task()) 2199 drain_obj_stock(&stock->task_obj); 2200 drain_stock(stock); 2201 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags); 2202 2203 local_irq_restore(flags); 2204 } 2205 2206 /* 2207 * Cache charges(val) to local per_cpu area. 2208 * This will be consumed by consume_stock() function, later. 2209 */ 2210 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) 2211 { 2212 struct memcg_stock_pcp *stock; 2213 unsigned long flags; 2214 2215 local_irq_save(flags); 2216 2217 stock = this_cpu_ptr(&memcg_stock); 2218 if (stock->cached != memcg) { /* reset if necessary */ 2219 drain_stock(stock); 2220 css_get(&memcg->css); 2221 stock->cached = memcg; 2222 } 2223 stock->nr_pages += nr_pages; 2224 2225 if (stock->nr_pages > MEMCG_CHARGE_BATCH) 2226 drain_stock(stock); 2227 2228 local_irq_restore(flags); 2229 } 2230 2231 /* 2232 * Drains all per-CPU charge caches for given root_memcg resp. subtree 2233 * of the hierarchy under it. 2234 */ 2235 static void drain_all_stock(struct mem_cgroup *root_memcg) 2236 { 2237 int cpu, curcpu; 2238 2239 /* If someone's already draining, avoid adding running more workers. */ 2240 if (!mutex_trylock(&percpu_charge_mutex)) 2241 return; 2242 /* 2243 * Notify other cpus that system-wide "drain" is running 2244 * We do not care about races with the cpu hotplug because cpu down 2245 * as well as workers from this path always operate on the local 2246 * per-cpu data. CPU up doesn't touch memcg_stock at all. 2247 */ 2248 curcpu = get_cpu(); 2249 for_each_online_cpu(cpu) { 2250 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); 2251 struct mem_cgroup *memcg; 2252 bool flush = false; 2253 2254 rcu_read_lock(); 2255 memcg = stock->cached; 2256 if (memcg && stock->nr_pages && 2257 mem_cgroup_is_descendant(memcg, root_memcg)) 2258 flush = true; 2259 else if (obj_stock_flush_required(stock, root_memcg)) 2260 flush = true; 2261 rcu_read_unlock(); 2262 2263 if (flush && 2264 !test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) { 2265 if (cpu == curcpu) 2266 drain_local_stock(&stock->work); 2267 else 2268 schedule_work_on(cpu, &stock->work); 2269 } 2270 } 2271 put_cpu(); 2272 mutex_unlock(&percpu_charge_mutex); 2273 } 2274 2275 static int memcg_hotplug_cpu_dead(unsigned int cpu) 2276 { 2277 struct memcg_stock_pcp *stock; 2278 2279 stock = &per_cpu(memcg_stock, cpu); 2280 drain_stock(stock); 2281 2282 return 0; 2283 } 2284 2285 static unsigned long reclaim_high(struct mem_cgroup *memcg, 2286 unsigned int nr_pages, 2287 gfp_t gfp_mask) 2288 { 2289 unsigned long nr_reclaimed = 0; 2290 2291 do { 2292 unsigned long pflags; 2293 2294 if (page_counter_read(&memcg->memory) <= 2295 READ_ONCE(memcg->memory.high)) 2296 continue; 2297 2298 memcg_memory_event(memcg, MEMCG_HIGH); 2299 2300 psi_memstall_enter(&pflags); 2301 nr_reclaimed += try_to_free_mem_cgroup_pages(memcg, nr_pages, 2302 gfp_mask, true); 2303 psi_memstall_leave(&pflags); 2304 } while ((memcg = parent_mem_cgroup(memcg)) && 2305 !mem_cgroup_is_root(memcg)); 2306 2307 return nr_reclaimed; 2308 } 2309 2310 static void high_work_func(struct work_struct *work) 2311 { 2312 struct mem_cgroup *memcg; 2313 2314 memcg = container_of(work, struct mem_cgroup, high_work); 2315 reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL); 2316 } 2317 2318 /* 2319 * Clamp the maximum sleep time per allocation batch to 2 seconds. This is 2320 * enough to still cause a significant slowdown in most cases, while still 2321 * allowing diagnostics and tracing to proceed without becoming stuck. 2322 */ 2323 #define MEMCG_MAX_HIGH_DELAY_JIFFIES (2UL*HZ) 2324 2325 /* 2326 * When calculating the delay, we use these either side of the exponentiation to 2327 * maintain precision and scale to a reasonable number of jiffies (see the table 2328 * below. 2329 * 2330 * - MEMCG_DELAY_PRECISION_SHIFT: Extra precision bits while translating the 2331 * overage ratio to a delay. 2332 * - MEMCG_DELAY_SCALING_SHIFT: The number of bits to scale down the 2333 * proposed penalty in order to reduce to a reasonable number of jiffies, and 2334 * to produce a reasonable delay curve. 2335 * 2336 * MEMCG_DELAY_SCALING_SHIFT just happens to be a number that produces a 2337 * reasonable delay curve compared to precision-adjusted overage, not 2338 * penalising heavily at first, but still making sure that growth beyond the 2339 * limit penalises misbehaviour cgroups by slowing them down exponentially. For 2340 * example, with a high of 100 megabytes: 2341 * 2342 * +-------+------------------------+ 2343 * | usage | time to allocate in ms | 2344 * +-------+------------------------+ 2345 * | 100M | 0 | 2346 * | 101M | 6 | 2347 * | 102M | 25 | 2348 * | 103M | 57 | 2349 * | 104M | 102 | 2350 * | 105M | 159 | 2351 * | 106M | 230 | 2352 * | 107M | 313 | 2353 * | 108M | 409 | 2354 * | 109M | 518 | 2355 * | 110M | 639 | 2356 * | 111M | 774 | 2357 * | 112M | 921 | 2358 * | 113M | 1081 | 2359 * | 114M | 1254 | 2360 * | 115M | 1439 | 2361 * | 116M | 1638 | 2362 * | 117M | 1849 | 2363 * | 118M | 2000 | 2364 * | 119M | 2000 | 2365 * | 120M | 2000 | 2366 * +-------+------------------------+ 2367 */ 2368 #define MEMCG_DELAY_PRECISION_SHIFT 20 2369 #define MEMCG_DELAY_SCALING_SHIFT 14 2370 2371 static u64 calculate_overage(unsigned long usage, unsigned long high) 2372 { 2373 u64 overage; 2374 2375 if (usage <= high) 2376 return 0; 2377 2378 /* 2379 * Prevent division by 0 in overage calculation by acting as if 2380 * it was a threshold of 1 page 2381 */ 2382 high = max(high, 1UL); 2383 2384 overage = usage - high; 2385 overage <<= MEMCG_DELAY_PRECISION_SHIFT; 2386 return div64_u64(overage, high); 2387 } 2388 2389 static u64 mem_find_max_overage(struct mem_cgroup *memcg) 2390 { 2391 u64 overage, max_overage = 0; 2392 2393 do { 2394 overage = calculate_overage(page_counter_read(&memcg->memory), 2395 READ_ONCE(memcg->memory.high)); 2396 max_overage = max(overage, max_overage); 2397 } while ((memcg = parent_mem_cgroup(memcg)) && 2398 !mem_cgroup_is_root(memcg)); 2399 2400 return max_overage; 2401 } 2402 2403 static u64 swap_find_max_overage(struct mem_cgroup *memcg) 2404 { 2405 u64 overage, max_overage = 0; 2406 2407 do { 2408 overage = calculate_overage(page_counter_read(&memcg->swap), 2409 READ_ONCE(memcg->swap.high)); 2410 if (overage) 2411 memcg_memory_event(memcg, MEMCG_SWAP_HIGH); 2412 max_overage = max(overage, max_overage); 2413 } while ((memcg = parent_mem_cgroup(memcg)) && 2414 !mem_cgroup_is_root(memcg)); 2415 2416 return max_overage; 2417 } 2418 2419 /* 2420 * Get the number of jiffies that we should penalise a mischievous cgroup which 2421 * is exceeding its memory.high by checking both it and its ancestors. 2422 */ 2423 static unsigned long calculate_high_delay(struct mem_cgroup *memcg, 2424 unsigned int nr_pages, 2425 u64 max_overage) 2426 { 2427 unsigned long penalty_jiffies; 2428 2429 if (!max_overage) 2430 return 0; 2431 2432 /* 2433 * We use overage compared to memory.high to calculate the number of 2434 * jiffies to sleep (penalty_jiffies). Ideally this value should be 2435 * fairly lenient on small overages, and increasingly harsh when the 2436 * memcg in question makes it clear that it has no intention of stopping 2437 * its crazy behaviour, so we exponentially increase the delay based on 2438 * overage amount. 2439 */ 2440 penalty_jiffies = max_overage * max_overage * HZ; 2441 penalty_jiffies >>= MEMCG_DELAY_PRECISION_SHIFT; 2442 penalty_jiffies >>= MEMCG_DELAY_SCALING_SHIFT; 2443 2444 /* 2445 * Factor in the task's own contribution to the overage, such that four 2446 * N-sized allocations are throttled approximately the same as one 2447 * 4N-sized allocation. 2448 * 2449 * MEMCG_CHARGE_BATCH pages is nominal, so work out how much smaller or 2450 * larger the current charge patch is than that. 2451 */ 2452 return penalty_jiffies * nr_pages / MEMCG_CHARGE_BATCH; 2453 } 2454 2455 /* 2456 * Scheduled by try_charge() to be executed from the userland return path 2457 * and reclaims memory over the high limit. 2458 */ 2459 void mem_cgroup_handle_over_high(void) 2460 { 2461 unsigned long penalty_jiffies; 2462 unsigned long pflags; 2463 unsigned long nr_reclaimed; 2464 unsigned int nr_pages = current->memcg_nr_pages_over_high; 2465 int nr_retries = MAX_RECLAIM_RETRIES; 2466 struct mem_cgroup *memcg; 2467 bool in_retry = false; 2468 2469 if (likely(!nr_pages)) 2470 return; 2471 2472 memcg = get_mem_cgroup_from_mm(current->mm); 2473 current->memcg_nr_pages_over_high = 0; 2474 2475 retry_reclaim: 2476 /* 2477 * The allocating task should reclaim at least the batch size, but for 2478 * subsequent retries we only want to do what's necessary to prevent oom 2479 * or breaching resource isolation. 2480 * 2481 * This is distinct from memory.max or page allocator behaviour because 2482 * memory.high is currently batched, whereas memory.max and the page 2483 * allocator run every time an allocation is made. 2484 */ 2485 nr_reclaimed = reclaim_high(memcg, 2486 in_retry ? SWAP_CLUSTER_MAX : nr_pages, 2487 GFP_KERNEL); 2488 2489 /* 2490 * memory.high is breached and reclaim is unable to keep up. Throttle 2491 * allocators proactively to slow down excessive growth. 2492 */ 2493 penalty_jiffies = calculate_high_delay(memcg, nr_pages, 2494 mem_find_max_overage(memcg)); 2495 2496 penalty_jiffies += calculate_high_delay(memcg, nr_pages, 2497 swap_find_max_overage(memcg)); 2498 2499 /* 2500 * Clamp the max delay per usermode return so as to still keep the 2501 * application moving forwards and also permit diagnostics, albeit 2502 * extremely slowly. 2503 */ 2504 penalty_jiffies = min(penalty_jiffies, MEMCG_MAX_HIGH_DELAY_JIFFIES); 2505 2506 /* 2507 * Don't sleep if the amount of jiffies this memcg owes us is so low 2508 * that it's not even worth doing, in an attempt to be nice to those who 2509 * go only a small amount over their memory.high value and maybe haven't 2510 * been aggressively reclaimed enough yet. 2511 */ 2512 if (penalty_jiffies <= HZ / 100) 2513 goto out; 2514 2515 /* 2516 * If reclaim is making forward progress but we're still over 2517 * memory.high, we want to encourage that rather than doing allocator 2518 * throttling. 2519 */ 2520 if (nr_reclaimed || nr_retries--) { 2521 in_retry = true; 2522 goto retry_reclaim; 2523 } 2524 2525 /* 2526 * If we exit early, we're guaranteed to die (since 2527 * schedule_timeout_killable sets TASK_KILLABLE). This means we don't 2528 * need to account for any ill-begotten jiffies to pay them off later. 2529 */ 2530 psi_memstall_enter(&pflags); 2531 schedule_timeout_killable(penalty_jiffies); 2532 psi_memstall_leave(&pflags); 2533 2534 out: 2535 css_put(&memcg->css); 2536 } 2537 2538 static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, 2539 unsigned int nr_pages) 2540 { 2541 unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages); 2542 int nr_retries = MAX_RECLAIM_RETRIES; 2543 struct mem_cgroup *mem_over_limit; 2544 struct page_counter *counter; 2545 enum oom_status oom_status; 2546 unsigned long nr_reclaimed; 2547 bool may_swap = true; 2548 bool drained = false; 2549 unsigned long pflags; 2550 2551 retry: 2552 if (consume_stock(memcg, nr_pages)) 2553 return 0; 2554 2555 if (!do_memsw_account() || 2556 page_counter_try_charge(&memcg->memsw, batch, &counter)) { 2557 if (page_counter_try_charge(&memcg->memory, batch, &counter)) 2558 goto done_restock; 2559 if (do_memsw_account()) 2560 page_counter_uncharge(&memcg->memsw, batch); 2561 mem_over_limit = mem_cgroup_from_counter(counter, memory); 2562 } else { 2563 mem_over_limit = mem_cgroup_from_counter(counter, memsw); 2564 may_swap = false; 2565 } 2566 2567 if (batch > nr_pages) { 2568 batch = nr_pages; 2569 goto retry; 2570 } 2571 2572 /* 2573 * Memcg doesn't have a dedicated reserve for atomic 2574 * allocations. But like the global atomic pool, we need to 2575 * put the burden of reclaim on regular allocation requests 2576 * and let these go through as privileged allocations. 2577 */ 2578 if (gfp_mask & __GFP_ATOMIC) 2579 goto force; 2580 2581 /* 2582 * Unlike in global OOM situations, memcg is not in a physical 2583 * memory shortage. Allow dying and OOM-killed tasks to 2584 * bypass the last charges so that they can exit quickly and 2585 * free their memory. 2586 */ 2587 if (unlikely(should_force_charge())) 2588 goto force; 2589 2590 /* 2591 * Prevent unbounded recursion when reclaim operations need to 2592 * allocate memory. This might exceed the limits temporarily, 2593 * but we prefer facilitating memory reclaim and getting back 2594 * under the limit over triggering OOM kills in these cases. 2595 */ 2596 if (unlikely(current->flags & PF_MEMALLOC)) 2597 goto force; 2598 2599 if (unlikely(task_in_memcg_oom(current))) 2600 goto nomem; 2601 2602 if (!gfpflags_allow_blocking(gfp_mask)) 2603 goto nomem; 2604 2605 memcg_memory_event(mem_over_limit, MEMCG_MAX); 2606 2607 psi_memstall_enter(&pflags); 2608 nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages, 2609 gfp_mask, may_swap); 2610 psi_memstall_leave(&pflags); 2611 2612 if (mem_cgroup_margin(mem_over_limit) >= nr_pages) 2613 goto retry; 2614 2615 if (!drained) { 2616 drain_all_stock(mem_over_limit); 2617 drained = true; 2618 goto retry; 2619 } 2620 2621 if (gfp_mask & __GFP_NORETRY) 2622 goto nomem; 2623 /* 2624 * Even though the limit is exceeded at this point, reclaim 2625 * may have been able to free some pages. Retry the charge 2626 * before killing the task. 2627 * 2628 * Only for regular pages, though: huge pages are rather 2629 * unlikely to succeed so close to the limit, and we fall back 2630 * to regular pages anyway in case of failure. 2631 */ 2632 if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER)) 2633 goto retry; 2634 /* 2635 * At task move, charge accounts can be doubly counted. So, it's 2636 * better to wait until the end of task_move if something is going on. 2637 */ 2638 if (mem_cgroup_wait_acct_move(mem_over_limit)) 2639 goto retry; 2640 2641 if (nr_retries--) 2642 goto retry; 2643 2644 if (gfp_mask & __GFP_RETRY_MAYFAIL) 2645 goto nomem; 2646 2647 if (fatal_signal_pending(current)) 2648 goto force; 2649 2650 /* 2651 * keep retrying as long as the memcg oom killer is able to make 2652 * a forward progress or bypass the charge if the oom killer 2653 * couldn't make any progress. 2654 */ 2655 oom_status = mem_cgroup_oom(mem_over_limit, gfp_mask, 2656 get_order(nr_pages * PAGE_SIZE)); 2657 switch (oom_status) { 2658 case OOM_SUCCESS: 2659 nr_retries = MAX_RECLAIM_RETRIES; 2660 goto retry; 2661 case OOM_FAILED: 2662 goto force; 2663 default: 2664 goto nomem; 2665 } 2666 nomem: 2667 if (!(gfp_mask & __GFP_NOFAIL)) 2668 return -ENOMEM; 2669 force: 2670 /* 2671 * The allocation either can't fail or will lead to more memory 2672 * being freed very soon. Allow memory usage go over the limit 2673 * temporarily by force charging it. 2674 */ 2675 page_counter_charge(&memcg->memory, nr_pages); 2676 if (do_memsw_account()) 2677 page_counter_charge(&memcg->memsw, nr_pages); 2678 2679 return 0; 2680 2681 done_restock: 2682 if (batch > nr_pages) 2683 refill_stock(memcg, batch - nr_pages); 2684 2685 /* 2686 * If the hierarchy is above the normal consumption range, schedule 2687 * reclaim on returning to userland. We can perform reclaim here 2688 * if __GFP_RECLAIM but let's always punt for simplicity and so that 2689 * GFP_KERNEL can consistently be used during reclaim. @memcg is 2690 * not recorded as it most likely matches current's and won't 2691 * change in the meantime. As high limit is checked again before 2692 * reclaim, the cost of mismatch is negligible. 2693 */ 2694 do { 2695 bool mem_high, swap_high; 2696 2697 mem_high = page_counter_read(&memcg->memory) > 2698 READ_ONCE(memcg->memory.high); 2699 swap_high = page_counter_read(&memcg->swap) > 2700 READ_ONCE(memcg->swap.high); 2701 2702 /* Don't bother a random interrupted task */ 2703 if (in_interrupt()) { 2704 if (mem_high) { 2705 schedule_work(&memcg->high_work); 2706 break; 2707 } 2708 continue; 2709 } 2710 2711 if (mem_high || swap_high) { 2712 /* 2713 * The allocating tasks in this cgroup will need to do 2714 * reclaim or be throttled to prevent further growth 2715 * of the memory or swap footprints. 2716 * 2717 * Target some best-effort fairness between the tasks, 2718 * and distribute reclaim work and delay penalties 2719 * based on how much each task is actually allocating. 2720 */ 2721 current->memcg_nr_pages_over_high += batch; 2722 set_notify_resume(current); 2723 break; 2724 } 2725 } while ((memcg = parent_mem_cgroup(memcg))); 2726 2727 return 0; 2728 } 2729 2730 static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, 2731 unsigned int nr_pages) 2732 { 2733 if (mem_cgroup_is_root(memcg)) 2734 return 0; 2735 2736 return try_charge_memcg(memcg, gfp_mask, nr_pages); 2737 } 2738 2739 #if defined(CONFIG_MEMCG_KMEM) || defined(CONFIG_MMU) 2740 static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages) 2741 { 2742 if (mem_cgroup_is_root(memcg)) 2743 return; 2744 2745 page_counter_uncharge(&memcg->memory, nr_pages); 2746 if (do_memsw_account()) 2747 page_counter_uncharge(&memcg->memsw, nr_pages); 2748 } 2749 #endif 2750 2751 static void commit_charge(struct folio *folio, struct mem_cgroup *memcg) 2752 { 2753 VM_BUG_ON_FOLIO(folio_memcg(folio), folio); 2754 /* 2755 * Any of the following ensures page's memcg stability: 2756 * 2757 * - the page lock 2758 * - LRU isolation 2759 * - lock_page_memcg() 2760 * - exclusive reference 2761 */ 2762 folio->memcg_data = (unsigned long)memcg; 2763 } 2764 2765 static struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *objcg) 2766 { 2767 struct mem_cgroup *memcg; 2768 2769 rcu_read_lock(); 2770 retry: 2771 memcg = obj_cgroup_memcg(objcg); 2772 if (unlikely(!css_tryget(&memcg->css))) 2773 goto retry; 2774 rcu_read_unlock(); 2775 2776 return memcg; 2777 } 2778 2779 #ifdef CONFIG_MEMCG_KMEM 2780 /* 2781 * The allocated objcg pointers array is not accounted directly. 2782 * Moreover, it should not come from DMA buffer and is not readily 2783 * reclaimable. So those GFP bits should be masked off. 2784 */ 2785 #define OBJCGS_CLEAR_MASK (__GFP_DMA | __GFP_RECLAIMABLE | __GFP_ACCOUNT) 2786 2787 int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s, 2788 gfp_t gfp, bool new_page) 2789 { 2790 unsigned int objects = objs_per_slab_page(s, page); 2791 unsigned long memcg_data; 2792 void *vec; 2793 2794 gfp &= ~OBJCGS_CLEAR_MASK; 2795 vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp, 2796 page_to_nid(page)); 2797 if (!vec) 2798 return -ENOMEM; 2799 2800 memcg_data = (unsigned long) vec | MEMCG_DATA_OBJCGS; 2801 if (new_page) { 2802 /* 2803 * If the slab page is brand new and nobody can yet access 2804 * it's memcg_data, no synchronization is required and 2805 * memcg_data can be simply assigned. 2806 */ 2807 page->memcg_data = memcg_data; 2808 } else if (cmpxchg(&page->memcg_data, 0, memcg_data)) { 2809 /* 2810 * If the slab page is already in use, somebody can allocate 2811 * and assign obj_cgroups in parallel. In this case the existing 2812 * objcg vector should be reused. 2813 */ 2814 kfree(vec); 2815 return 0; 2816 } 2817 2818 kmemleak_not_leak(vec); 2819 return 0; 2820 } 2821 2822 /* 2823 * Returns a pointer to the memory cgroup to which the kernel object is charged. 2824 * 2825 * A passed kernel object can be a slab object or a generic kernel page, so 2826 * different mechanisms for getting the memory cgroup pointer should be used. 2827 * In certain cases (e.g. kernel stacks or large kmallocs with SLUB) the caller 2828 * can not know for sure how the kernel object is implemented. 2829 * mem_cgroup_from_obj() can be safely used in such cases. 2830 * 2831 * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(), 2832 * cgroup_mutex, etc. 2833 */ 2834 struct mem_cgroup *mem_cgroup_from_obj(void *p) 2835 { 2836 struct page *page; 2837 2838 if (mem_cgroup_disabled()) 2839 return NULL; 2840 2841 page = virt_to_head_page(p); 2842 2843 /* 2844 * Slab objects are accounted individually, not per-page. 2845 * Memcg membership data for each individual object is saved in 2846 * the page->obj_cgroups. 2847 */ 2848 if (page_objcgs_check(page)) { 2849 struct obj_cgroup *objcg; 2850 unsigned int off; 2851 2852 off = obj_to_index(page->slab_cache, page, p); 2853 objcg = page_objcgs(page)[off]; 2854 if (objcg) 2855 return obj_cgroup_memcg(objcg); 2856 2857 return NULL; 2858 } 2859 2860 /* 2861 * page_memcg_check() is used here, because page_has_obj_cgroups() 2862 * check above could fail because the object cgroups vector wasn't set 2863 * at that moment, but it can be set concurrently. 2864 * page_memcg_check(page) will guarantee that a proper memory 2865 * cgroup pointer or NULL will be returned. 2866 */ 2867 return page_memcg_check(page); 2868 } 2869 2870 __always_inline struct obj_cgroup *get_obj_cgroup_from_current(void) 2871 { 2872 struct obj_cgroup *objcg = NULL; 2873 struct mem_cgroup *memcg; 2874 2875 if (memcg_kmem_bypass()) 2876 return NULL; 2877 2878 rcu_read_lock(); 2879 if (unlikely(active_memcg())) 2880 memcg = active_memcg(); 2881 else 2882 memcg = mem_cgroup_from_task(current); 2883 2884 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) { 2885 objcg = rcu_dereference(memcg->objcg); 2886 if (objcg && obj_cgroup_tryget(objcg)) 2887 break; 2888 objcg = NULL; 2889 } 2890 rcu_read_unlock(); 2891 2892 return objcg; 2893 } 2894 2895 static int memcg_alloc_cache_id(void) 2896 { 2897 int id, size; 2898 int err; 2899 2900 id = ida_simple_get(&memcg_cache_ida, 2901 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL); 2902 if (id < 0) 2903 return id; 2904 2905 if (id < memcg_nr_cache_ids) 2906 return id; 2907 2908 /* 2909 * There's no space for the new id in memcg_caches arrays, 2910 * so we have to grow them. 2911 */ 2912 down_write(&memcg_cache_ids_sem); 2913 2914 size = 2 * (id + 1); 2915 if (size < MEMCG_CACHES_MIN_SIZE) 2916 size = MEMCG_CACHES_MIN_SIZE; 2917 else if (size > MEMCG_CACHES_MAX_SIZE) 2918 size = MEMCG_CACHES_MAX_SIZE; 2919 2920 err = memcg_update_all_list_lrus(size); 2921 if (!err) 2922 memcg_nr_cache_ids = size; 2923 2924 up_write(&memcg_cache_ids_sem); 2925 2926 if (err) { 2927 ida_simple_remove(&memcg_cache_ida, id); 2928 return err; 2929 } 2930 return id; 2931 } 2932 2933 static void memcg_free_cache_id(int id) 2934 { 2935 ida_simple_remove(&memcg_cache_ida, id); 2936 } 2937 2938 /* 2939 * obj_cgroup_uncharge_pages: uncharge a number of kernel pages from a objcg 2940 * @objcg: object cgroup to uncharge 2941 * @nr_pages: number of pages to uncharge 2942 */ 2943 static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg, 2944 unsigned int nr_pages) 2945 { 2946 struct mem_cgroup *memcg; 2947 2948 memcg = get_mem_cgroup_from_objcg(objcg); 2949 2950 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) 2951 page_counter_uncharge(&memcg->kmem, nr_pages); 2952 refill_stock(memcg, nr_pages); 2953 2954 css_put(&memcg->css); 2955 } 2956 2957 /* 2958 * obj_cgroup_charge_pages: charge a number of kernel pages to a objcg 2959 * @objcg: object cgroup to charge 2960 * @gfp: reclaim mode 2961 * @nr_pages: number of pages to charge 2962 * 2963 * Returns 0 on success, an error code on failure. 2964 */ 2965 static int obj_cgroup_charge_pages(struct obj_cgroup *objcg, gfp_t gfp, 2966 unsigned int nr_pages) 2967 { 2968 struct page_counter *counter; 2969 struct mem_cgroup *memcg; 2970 int ret; 2971 2972 memcg = get_mem_cgroup_from_objcg(objcg); 2973 2974 ret = try_charge_memcg(memcg, gfp, nr_pages); 2975 if (ret) 2976 goto out; 2977 2978 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && 2979 !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) { 2980 2981 /* 2982 * Enforce __GFP_NOFAIL allocation because callers are not 2983 * prepared to see failures and likely do not have any failure 2984 * handling code. 2985 */ 2986 if (gfp & __GFP_NOFAIL) { 2987 page_counter_charge(&memcg->kmem, nr_pages); 2988 goto out; 2989 } 2990 cancel_charge(memcg, nr_pages); 2991 ret = -ENOMEM; 2992 } 2993 out: 2994 css_put(&memcg->css); 2995 2996 return ret; 2997 } 2998 2999 /** 3000 * __memcg_kmem_charge_page: charge a kmem page to the current memory cgroup 3001 * @page: page to charge 3002 * @gfp: reclaim mode 3003 * @order: allocation order 3004 * 3005 * Returns 0 on success, an error code on failure. 3006 */ 3007 int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order) 3008 { 3009 struct obj_cgroup *objcg; 3010 int ret = 0; 3011 3012 objcg = get_obj_cgroup_from_current(); 3013 if (objcg) { 3014 ret = obj_cgroup_charge_pages(objcg, gfp, 1 << order); 3015 if (!ret) { 3016 page->memcg_data = (unsigned long)objcg | 3017 MEMCG_DATA_KMEM; 3018 return 0; 3019 } 3020 obj_cgroup_put(objcg); 3021 } 3022 return ret; 3023 } 3024 3025 /** 3026 * __memcg_kmem_uncharge_page: uncharge a kmem page 3027 * @page: page to uncharge 3028 * @order: allocation order 3029 */ 3030 void __memcg_kmem_uncharge_page(struct page *page, int order) 3031 { 3032 struct folio *folio = page_folio(page); 3033 struct obj_cgroup *objcg; 3034 unsigned int nr_pages = 1 << order; 3035 3036 if (!folio_memcg_kmem(folio)) 3037 return; 3038 3039 objcg = __folio_objcg(folio); 3040 obj_cgroup_uncharge_pages(objcg, nr_pages); 3041 folio->memcg_data = 0; 3042 obj_cgroup_put(objcg); 3043 } 3044 3045 void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat, 3046 enum node_stat_item idx, int nr) 3047 { 3048 unsigned long flags; 3049 struct obj_stock *stock = get_obj_stock(&flags); 3050 int *bytes; 3051 3052 /* 3053 * Save vmstat data in stock and skip vmstat array update unless 3054 * accumulating over a page of vmstat data or when pgdat or idx 3055 * changes. 3056 */ 3057 if (stock->cached_objcg != objcg) { 3058 drain_obj_stock(stock); 3059 obj_cgroup_get(objcg); 3060 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes) 3061 ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0; 3062 stock->cached_objcg = objcg; 3063 stock->cached_pgdat = pgdat; 3064 } else if (stock->cached_pgdat != pgdat) { 3065 /* Flush the existing cached vmstat data */ 3066 struct pglist_data *oldpg = stock->cached_pgdat; 3067 3068 if (stock->nr_slab_reclaimable_b) { 3069 mod_objcg_mlstate(objcg, oldpg, NR_SLAB_RECLAIMABLE_B, 3070 stock->nr_slab_reclaimable_b); 3071 stock->nr_slab_reclaimable_b = 0; 3072 } 3073 if (stock->nr_slab_unreclaimable_b) { 3074 mod_objcg_mlstate(objcg, oldpg, NR_SLAB_UNRECLAIMABLE_B, 3075 stock->nr_slab_unreclaimable_b); 3076 stock->nr_slab_unreclaimable_b = 0; 3077 } 3078 stock->cached_pgdat = pgdat; 3079 } 3080 3081 bytes = (idx == NR_SLAB_RECLAIMABLE_B) ? &stock->nr_slab_reclaimable_b 3082 : &stock->nr_slab_unreclaimable_b; 3083 /* 3084 * Even for large object >= PAGE_SIZE, the vmstat data will still be 3085 * cached locally at least once before pushing it out. 3086 */ 3087 if (!*bytes) { 3088 *bytes = nr; 3089 nr = 0; 3090 } else { 3091 *bytes += nr; 3092 if (abs(*bytes) > PAGE_SIZE) { 3093 nr = *bytes; 3094 *bytes = 0; 3095 } else { 3096 nr = 0; 3097 } 3098 } 3099 if (nr) 3100 mod_objcg_mlstate(objcg, pgdat, idx, nr); 3101 3102 put_obj_stock(flags); 3103 } 3104 3105 static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes) 3106 { 3107 unsigned long flags; 3108 struct obj_stock *stock = get_obj_stock(&flags); 3109 bool ret = false; 3110 3111 if (objcg == stock->cached_objcg && stock->nr_bytes >= nr_bytes) { 3112 stock->nr_bytes -= nr_bytes; 3113 ret = true; 3114 } 3115 3116 put_obj_stock(flags); 3117 3118 return ret; 3119 } 3120 3121 static void drain_obj_stock(struct obj_stock *stock) 3122 { 3123 struct obj_cgroup *old = stock->cached_objcg; 3124 3125 if (!old) 3126 return; 3127 3128 if (stock->nr_bytes) { 3129 unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT; 3130 unsigned int nr_bytes = stock->nr_bytes & (PAGE_SIZE - 1); 3131 3132 if (nr_pages) 3133 obj_cgroup_uncharge_pages(old, nr_pages); 3134 3135 /* 3136 * The leftover is flushed to the centralized per-memcg value. 3137 * On the next attempt to refill obj stock it will be moved 3138 * to a per-cpu stock (probably, on an other CPU), see 3139 * refill_obj_stock(). 3140 * 3141 * How often it's flushed is a trade-off between the memory 3142 * limit enforcement accuracy and potential CPU contention, 3143 * so it might be changed in the future. 3144 */ 3145 atomic_add(nr_bytes, &old->nr_charged_bytes); 3146 stock->nr_bytes = 0; 3147 } 3148 3149 /* 3150 * Flush the vmstat data in current stock 3151 */ 3152 if (stock->nr_slab_reclaimable_b || stock->nr_slab_unreclaimable_b) { 3153 if (stock->nr_slab_reclaimable_b) { 3154 mod_objcg_mlstate(old, stock->cached_pgdat, 3155 NR_SLAB_RECLAIMABLE_B, 3156 stock->nr_slab_reclaimable_b); 3157 stock->nr_slab_reclaimable_b = 0; 3158 } 3159 if (stock->nr_slab_unreclaimable_b) { 3160 mod_objcg_mlstate(old, stock->cached_pgdat, 3161 NR_SLAB_UNRECLAIMABLE_B, 3162 stock->nr_slab_unreclaimable_b); 3163 stock->nr_slab_unreclaimable_b = 0; 3164 } 3165 stock->cached_pgdat = NULL; 3166 } 3167 3168 obj_cgroup_put(old); 3169 stock->cached_objcg = NULL; 3170 } 3171 3172 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock, 3173 struct mem_cgroup *root_memcg) 3174 { 3175 struct mem_cgroup *memcg; 3176 3177 if (in_task() && stock->task_obj.cached_objcg) { 3178 memcg = obj_cgroup_memcg(stock->task_obj.cached_objcg); 3179 if (memcg && mem_cgroup_is_descendant(memcg, root_memcg)) 3180 return true; 3181 } 3182 if (stock->irq_obj.cached_objcg) { 3183 memcg = obj_cgroup_memcg(stock->irq_obj.cached_objcg); 3184 if (memcg && mem_cgroup_is_descendant(memcg, root_memcg)) 3185 return true; 3186 } 3187 3188 return false; 3189 } 3190 3191 static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes, 3192 bool allow_uncharge) 3193 { 3194 unsigned long flags; 3195 struct obj_stock *stock = get_obj_stock(&flags); 3196 unsigned int nr_pages = 0; 3197 3198 if (stock->cached_objcg != objcg) { /* reset if necessary */ 3199 drain_obj_stock(stock); 3200 obj_cgroup_get(objcg); 3201 stock->cached_objcg = objcg; 3202 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes) 3203 ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0; 3204 allow_uncharge = true; /* Allow uncharge when objcg changes */ 3205 } 3206 stock->nr_bytes += nr_bytes; 3207 3208 if (allow_uncharge && (stock->nr_bytes > PAGE_SIZE)) { 3209 nr_pages = stock->nr_bytes >> PAGE_SHIFT; 3210 stock->nr_bytes &= (PAGE_SIZE - 1); 3211 } 3212 3213 put_obj_stock(flags); 3214 3215 if (nr_pages) 3216 obj_cgroup_uncharge_pages(objcg, nr_pages); 3217 } 3218 3219 int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size) 3220 { 3221 unsigned int nr_pages, nr_bytes; 3222 int ret; 3223 3224 if (consume_obj_stock(objcg, size)) 3225 return 0; 3226 3227 /* 3228 * In theory, objcg->nr_charged_bytes can have enough 3229 * pre-charged bytes to satisfy the allocation. However, 3230 * flushing objcg->nr_charged_bytes requires two atomic 3231 * operations, and objcg->nr_charged_bytes can't be big. 3232 * The shared objcg->nr_charged_bytes can also become a 3233 * performance bottleneck if all tasks of the same memcg are 3234 * trying to update it. So it's better to ignore it and try 3235 * grab some new pages. The stock's nr_bytes will be flushed to 3236 * objcg->nr_charged_bytes later on when objcg changes. 3237 * 3238 * The stock's nr_bytes may contain enough pre-charged bytes 3239 * to allow one less page from being charged, but we can't rely 3240 * on the pre-charged bytes not being changed outside of 3241 * consume_obj_stock() or refill_obj_stock(). So ignore those 3242 * pre-charged bytes as well when charging pages. To avoid a 3243 * page uncharge right after a page charge, we set the 3244 * allow_uncharge flag to false when calling refill_obj_stock() 3245 * to temporarily allow the pre-charged bytes to exceed the page 3246 * size limit. The maximum reachable value of the pre-charged 3247 * bytes is (sizeof(object) + PAGE_SIZE - 2) if there is no data 3248 * race. 3249 */ 3250 nr_pages = size >> PAGE_SHIFT; 3251 nr_bytes = size & (PAGE_SIZE - 1); 3252 3253 if (nr_bytes) 3254 nr_pages += 1; 3255 3256 ret = obj_cgroup_charge_pages(objcg, gfp, nr_pages); 3257 if (!ret && nr_bytes) 3258 refill_obj_stock(objcg, PAGE_SIZE - nr_bytes, false); 3259 3260 return ret; 3261 } 3262 3263 void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size) 3264 { 3265 refill_obj_stock(objcg, size, true); 3266 } 3267 3268 #endif /* CONFIG_MEMCG_KMEM */ 3269 3270 /* 3271 * Because page_memcg(head) is not set on tails, set it now. 3272 */ 3273 void split_page_memcg(struct page *head, unsigned int nr) 3274 { 3275 struct folio *folio = page_folio(head); 3276 struct mem_cgroup *memcg = folio_memcg(folio); 3277 int i; 3278 3279 if (mem_cgroup_disabled() || !memcg) 3280 return; 3281 3282 for (i = 1; i < nr; i++) 3283 folio_page(folio, i)->memcg_data = folio->memcg_data; 3284 3285 if (folio_memcg_kmem(folio)) 3286 obj_cgroup_get_many(__folio_objcg(folio), nr - 1); 3287 else 3288 css_get_many(&memcg->css, nr - 1); 3289 } 3290 3291 #ifdef CONFIG_MEMCG_SWAP 3292 /** 3293 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record. 3294 * @entry: swap entry to be moved 3295 * @from: mem_cgroup which the entry is moved from 3296 * @to: mem_cgroup which the entry is moved to 3297 * 3298 * It succeeds only when the swap_cgroup's record for this entry is the same 3299 * as the mem_cgroup's id of @from. 3300 * 3301 * Returns 0 on success, -EINVAL on failure. 3302 * 3303 * The caller must have charged to @to, IOW, called page_counter_charge() about 3304 * both res and memsw, and called css_get(). 3305 */ 3306 static int mem_cgroup_move_swap_account(swp_entry_t entry, 3307 struct mem_cgroup *from, struct mem_cgroup *to) 3308 { 3309 unsigned short old_id, new_id; 3310 3311 old_id = mem_cgroup_id(from); 3312 new_id = mem_cgroup_id(to); 3313 3314 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) { 3315 mod_memcg_state(from, MEMCG_SWAP, -1); 3316 mod_memcg_state(to, MEMCG_SWAP, 1); 3317 return 0; 3318 } 3319 return -EINVAL; 3320 } 3321 #else 3322 static inline int mem_cgroup_move_swap_account(swp_entry_t entry, 3323 struct mem_cgroup *from, struct mem_cgroup *to) 3324 { 3325 return -EINVAL; 3326 } 3327 #endif 3328 3329 static DEFINE_MUTEX(memcg_max_mutex); 3330 3331 static int mem_cgroup_resize_max(struct mem_cgroup *memcg, 3332 unsigned long max, bool memsw) 3333 { 3334 bool enlarge = false; 3335 bool drained = false; 3336 int ret; 3337 bool limits_invariant; 3338 struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory; 3339 3340 do { 3341 if (signal_pending(current)) { 3342 ret = -EINTR; 3343 break; 3344 } 3345 3346 mutex_lock(&memcg_max_mutex); 3347 /* 3348 * Make sure that the new limit (memsw or memory limit) doesn't 3349 * break our basic invariant rule memory.max <= memsw.max. 3350 */ 3351 limits_invariant = memsw ? max >= READ_ONCE(memcg->memory.max) : 3352 max <= memcg->memsw.max; 3353 if (!limits_invariant) { 3354 mutex_unlock(&memcg_max_mutex); 3355 ret = -EINVAL; 3356 break; 3357 } 3358 if (max > counter->max) 3359 enlarge = true; 3360 ret = page_counter_set_max(counter, max); 3361 mutex_unlock(&memcg_max_mutex); 3362 3363 if (!ret) 3364 break; 3365 3366 if (!drained) { 3367 drain_all_stock(memcg); 3368 drained = true; 3369 continue; 3370 } 3371 3372 if (!try_to_free_mem_cgroup_pages(memcg, 1, 3373 GFP_KERNEL, !memsw)) { 3374 ret = -EBUSY; 3375 break; 3376 } 3377 } while (true); 3378 3379 if (!ret && enlarge) 3380 memcg_oom_recover(memcg); 3381 3382 return ret; 3383 } 3384 3385 unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order, 3386 gfp_t gfp_mask, 3387 unsigned long *total_scanned) 3388 { 3389 unsigned long nr_reclaimed = 0; 3390 struct mem_cgroup_per_node *mz, *next_mz = NULL; 3391 unsigned long reclaimed; 3392 int loop = 0; 3393 struct mem_cgroup_tree_per_node *mctz; 3394 unsigned long excess; 3395 unsigned long nr_scanned; 3396 3397 if (order > 0) 3398 return 0; 3399 3400 mctz = soft_limit_tree.rb_tree_per_node[pgdat->node_id]; 3401 3402 /* 3403 * Do not even bother to check the largest node if the root 3404 * is empty. Do it lockless to prevent lock bouncing. Races 3405 * are acceptable as soft limit is best effort anyway. 3406 */ 3407 if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root)) 3408 return 0; 3409 3410 /* 3411 * This loop can run a while, specially if mem_cgroup's continuously 3412 * keep exceeding their soft limit and putting the system under 3413 * pressure 3414 */ 3415 do { 3416 if (next_mz) 3417 mz = next_mz; 3418 else 3419 mz = mem_cgroup_largest_soft_limit_node(mctz); 3420 if (!mz) 3421 break; 3422 3423 nr_scanned = 0; 3424 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat, 3425 gfp_mask, &nr_scanned); 3426 nr_reclaimed += reclaimed; 3427 *total_scanned += nr_scanned; 3428 spin_lock_irq(&mctz->lock); 3429 __mem_cgroup_remove_exceeded(mz, mctz); 3430 3431 /* 3432 * If we failed to reclaim anything from this memory cgroup 3433 * it is time to move on to the next cgroup 3434 */ 3435 next_mz = NULL; 3436 if (!reclaimed) 3437 next_mz = __mem_cgroup_largest_soft_limit_node(mctz); 3438 3439 excess = soft_limit_excess(mz->memcg); 3440 /* 3441 * One school of thought says that we should not add 3442 * back the node to the tree if reclaim returns 0. 3443 * But our reclaim could return 0, simply because due 3444 * to priority we are exposing a smaller subset of 3445 * memory to reclaim from. Consider this as a longer 3446 * term TODO. 3447 */ 3448 /* If excess == 0, no tree ops */ 3449 __mem_cgroup_insert_exceeded(mz, mctz, excess); 3450 spin_unlock_irq(&mctz->lock); 3451 css_put(&mz->memcg->css); 3452 loop++; 3453 /* 3454 * Could not reclaim anything and there are no more 3455 * mem cgroups to try or we seem to be looping without 3456 * reclaiming anything. 3457 */ 3458 if (!nr_reclaimed && 3459 (next_mz == NULL || 3460 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS)) 3461 break; 3462 } while (!nr_reclaimed); 3463 if (next_mz) 3464 css_put(&next_mz->memcg->css); 3465 return nr_reclaimed; 3466 } 3467 3468 /* 3469 * Reclaims as many pages from the given memcg as possible. 3470 * 3471 * Caller is responsible for holding css reference for memcg. 3472 */ 3473 static int mem_cgroup_force_empty(struct mem_cgroup *memcg) 3474 { 3475 int nr_retries = MAX_RECLAIM_RETRIES; 3476 3477 /* we call try-to-free pages for make this cgroup empty */ 3478 lru_add_drain_all(); 3479 3480 drain_all_stock(memcg); 3481 3482 /* try to free all pages in this cgroup */ 3483 while (nr_retries && page_counter_read(&memcg->memory)) { 3484 int progress; 3485 3486 if (signal_pending(current)) 3487 return -EINTR; 3488 3489 progress = try_to_free_mem_cgroup_pages(memcg, 1, 3490 GFP_KERNEL, true); 3491 if (!progress) { 3492 nr_retries--; 3493 /* maybe some writeback is necessary */ 3494 congestion_wait(BLK_RW_ASYNC, HZ/10); 3495 } 3496 3497 } 3498 3499 return 0; 3500 } 3501 3502 static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of, 3503 char *buf, size_t nbytes, 3504 loff_t off) 3505 { 3506 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 3507 3508 if (mem_cgroup_is_root(memcg)) 3509 return -EINVAL; 3510 return mem_cgroup_force_empty(memcg) ?: nbytes; 3511 } 3512 3513 static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css, 3514 struct cftype *cft) 3515 { 3516 return 1; 3517 } 3518 3519 static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css, 3520 struct cftype *cft, u64 val) 3521 { 3522 if (val == 1) 3523 return 0; 3524 3525 pr_warn_once("Non-hierarchical mode is deprecated. " 3526 "Please report your usecase to linux-mm@kvack.org if you " 3527 "depend on this functionality.\n"); 3528 3529 return -EINVAL; 3530 } 3531 3532 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap) 3533 { 3534 unsigned long val; 3535 3536 if (mem_cgroup_is_root(memcg)) { 3537 /* mem_cgroup_threshold() calls here from irqsafe context */ 3538 cgroup_rstat_flush_irqsafe(memcg->css.cgroup); 3539 val = memcg_page_state(memcg, NR_FILE_PAGES) + 3540 memcg_page_state(memcg, NR_ANON_MAPPED); 3541 if (swap) 3542 val += memcg_page_state(memcg, MEMCG_SWAP); 3543 } else { 3544 if (!swap) 3545 val = page_counter_read(&memcg->memory); 3546 else 3547 val = page_counter_read(&memcg->memsw); 3548 } 3549 return val; 3550 } 3551 3552 enum { 3553 RES_USAGE, 3554 RES_LIMIT, 3555 RES_MAX_USAGE, 3556 RES_FAILCNT, 3557 RES_SOFT_LIMIT, 3558 }; 3559 3560 static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css, 3561 struct cftype *cft) 3562 { 3563 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 3564 struct page_counter *counter; 3565 3566 switch (MEMFILE_TYPE(cft->private)) { 3567 case _MEM: 3568 counter = &memcg->memory; 3569 break; 3570 case _MEMSWAP: 3571 counter = &memcg->memsw; 3572 break; 3573 case _KMEM: 3574 counter = &memcg->kmem; 3575 break; 3576 case _TCP: 3577 counter = &memcg->tcpmem; 3578 break; 3579 default: 3580 BUG(); 3581 } 3582 3583 switch (MEMFILE_ATTR(cft->private)) { 3584 case RES_USAGE: 3585 if (counter == &memcg->memory) 3586 return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE; 3587 if (counter == &memcg->memsw) 3588 return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE; 3589 return (u64)page_counter_read(counter) * PAGE_SIZE; 3590 case RES_LIMIT: 3591 return (u64)counter->max * PAGE_SIZE; 3592 case RES_MAX_USAGE: 3593 return (u64)counter->watermark * PAGE_SIZE; 3594 case RES_FAILCNT: 3595 return counter->failcnt; 3596 case RES_SOFT_LIMIT: 3597 return (u64)memcg->soft_limit * PAGE_SIZE; 3598 default: 3599 BUG(); 3600 } 3601 } 3602 3603 #ifdef CONFIG_MEMCG_KMEM 3604 static int memcg_online_kmem(struct mem_cgroup *memcg) 3605 { 3606 struct obj_cgroup *objcg; 3607 int memcg_id; 3608 3609 if (cgroup_memory_nokmem) 3610 return 0; 3611 3612 BUG_ON(memcg->kmemcg_id >= 0); 3613 BUG_ON(memcg->kmem_state); 3614 3615 memcg_id = memcg_alloc_cache_id(); 3616 if (memcg_id < 0) 3617 return memcg_id; 3618 3619 objcg = obj_cgroup_alloc(); 3620 if (!objcg) { 3621 memcg_free_cache_id(memcg_id); 3622 return -ENOMEM; 3623 } 3624 objcg->memcg = memcg; 3625 rcu_assign_pointer(memcg->objcg, objcg); 3626 3627 static_branch_enable(&memcg_kmem_enabled_key); 3628 3629 memcg->kmemcg_id = memcg_id; 3630 memcg->kmem_state = KMEM_ONLINE; 3631 3632 return 0; 3633 } 3634 3635 static void memcg_offline_kmem(struct mem_cgroup *memcg) 3636 { 3637 struct cgroup_subsys_state *css; 3638 struct mem_cgroup *parent, *child; 3639 int kmemcg_id; 3640 3641 if (memcg->kmem_state != KMEM_ONLINE) 3642 return; 3643 3644 memcg->kmem_state = KMEM_ALLOCATED; 3645 3646 parent = parent_mem_cgroup(memcg); 3647 if (!parent) 3648 parent = root_mem_cgroup; 3649 3650 memcg_reparent_objcgs(memcg, parent); 3651 3652 kmemcg_id = memcg->kmemcg_id; 3653 BUG_ON(kmemcg_id < 0); 3654 3655 /* 3656 * Change kmemcg_id of this cgroup and all its descendants to the 3657 * parent's id, and then move all entries from this cgroup's list_lrus 3658 * to ones of the parent. After we have finished, all list_lrus 3659 * corresponding to this cgroup are guaranteed to remain empty. The 3660 * ordering is imposed by list_lru_node->lock taken by 3661 * memcg_drain_all_list_lrus(). 3662 */ 3663 rcu_read_lock(); /* can be called from css_free w/o cgroup_mutex */ 3664 css_for_each_descendant_pre(css, &memcg->css) { 3665 child = mem_cgroup_from_css(css); 3666 BUG_ON(child->kmemcg_id != kmemcg_id); 3667 child->kmemcg_id = parent->kmemcg_id; 3668 } 3669 rcu_read_unlock(); 3670 3671 memcg_drain_all_list_lrus(kmemcg_id, parent); 3672 3673 memcg_free_cache_id(kmemcg_id); 3674 } 3675 3676 static void memcg_free_kmem(struct mem_cgroup *memcg) 3677 { 3678 /* css_alloc() failed, offlining didn't happen */ 3679 if (unlikely(memcg->kmem_state == KMEM_ONLINE)) 3680 memcg_offline_kmem(memcg); 3681 } 3682 #else 3683 static int memcg_online_kmem(struct mem_cgroup *memcg) 3684 { 3685 return 0; 3686 } 3687 static void memcg_offline_kmem(struct mem_cgroup *memcg) 3688 { 3689 } 3690 static void memcg_free_kmem(struct mem_cgroup *memcg) 3691 { 3692 } 3693 #endif /* CONFIG_MEMCG_KMEM */ 3694 3695 static int memcg_update_kmem_max(struct mem_cgroup *memcg, 3696 unsigned long max) 3697 { 3698 int ret; 3699 3700 mutex_lock(&memcg_max_mutex); 3701 ret = page_counter_set_max(&memcg->kmem, max); 3702 mutex_unlock(&memcg_max_mutex); 3703 return ret; 3704 } 3705 3706 static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max) 3707 { 3708 int ret; 3709 3710 mutex_lock(&memcg_max_mutex); 3711 3712 ret = page_counter_set_max(&memcg->tcpmem, max); 3713 if (ret) 3714 goto out; 3715 3716 if (!memcg->tcpmem_active) { 3717 /* 3718 * The active flag needs to be written after the static_key 3719 * update. This is what guarantees that the socket activation 3720 * function is the last one to run. See mem_cgroup_sk_alloc() 3721 * for details, and note that we don't mark any socket as 3722 * belonging to this memcg until that flag is up. 3723 * 3724 * We need to do this, because static_keys will span multiple 3725 * sites, but we can't control their order. If we mark a socket 3726 * as accounted, but the accounting functions are not patched in 3727 * yet, we'll lose accounting. 3728 * 3729 * We never race with the readers in mem_cgroup_sk_alloc(), 3730 * because when this value change, the code to process it is not 3731 * patched in yet. 3732 */ 3733 static_branch_inc(&memcg_sockets_enabled_key); 3734 memcg->tcpmem_active = true; 3735 } 3736 out: 3737 mutex_unlock(&memcg_max_mutex); 3738 return ret; 3739 } 3740 3741 /* 3742 * The user of this function is... 3743 * RES_LIMIT. 3744 */ 3745 static ssize_t mem_cgroup_write(struct kernfs_open_file *of, 3746 char *buf, size_t nbytes, loff_t off) 3747 { 3748 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 3749 unsigned long nr_pages; 3750 int ret; 3751 3752 buf = strstrip(buf); 3753 ret = page_counter_memparse(buf, "-1", &nr_pages); 3754 if (ret) 3755 return ret; 3756 3757 switch (MEMFILE_ATTR(of_cft(of)->private)) { 3758 case RES_LIMIT: 3759 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */ 3760 ret = -EINVAL; 3761 break; 3762 } 3763 switch (MEMFILE_TYPE(of_cft(of)->private)) { 3764 case _MEM: 3765 ret = mem_cgroup_resize_max(memcg, nr_pages, false); 3766 break; 3767 case _MEMSWAP: 3768 ret = mem_cgroup_resize_max(memcg, nr_pages, true); 3769 break; 3770 case _KMEM: 3771 pr_warn_once("kmem.limit_in_bytes is deprecated and will be removed. " 3772 "Please report your usecase to linux-mm@kvack.org if you " 3773 "depend on this functionality.\n"); 3774 ret = memcg_update_kmem_max(memcg, nr_pages); 3775 break; 3776 case _TCP: 3777 ret = memcg_update_tcp_max(memcg, nr_pages); 3778 break; 3779 } 3780 break; 3781 case RES_SOFT_LIMIT: 3782 memcg->soft_limit = nr_pages; 3783 ret = 0; 3784 break; 3785 } 3786 return ret ?: nbytes; 3787 } 3788 3789 static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf, 3790 size_t nbytes, loff_t off) 3791 { 3792 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 3793 struct page_counter *counter; 3794 3795 switch (MEMFILE_TYPE(of_cft(of)->private)) { 3796 case _MEM: 3797 counter = &memcg->memory; 3798 break; 3799 case _MEMSWAP: 3800 counter = &memcg->memsw; 3801 break; 3802 case _KMEM: 3803 counter = &memcg->kmem; 3804 break; 3805 case _TCP: 3806 counter = &memcg->tcpmem; 3807 break; 3808 default: 3809 BUG(); 3810 } 3811 3812 switch (MEMFILE_ATTR(of_cft(of)->private)) { 3813 case RES_MAX_USAGE: 3814 page_counter_reset_watermark(counter); 3815 break; 3816 case RES_FAILCNT: 3817 counter->failcnt = 0; 3818 break; 3819 default: 3820 BUG(); 3821 } 3822 3823 return nbytes; 3824 } 3825 3826 static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css, 3827 struct cftype *cft) 3828 { 3829 return mem_cgroup_from_css(css)->move_charge_at_immigrate; 3830 } 3831 3832 #ifdef CONFIG_MMU 3833 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css, 3834 struct cftype *cft, u64 val) 3835 { 3836 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 3837 3838 if (val & ~MOVE_MASK) 3839 return -EINVAL; 3840 3841 /* 3842 * No kind of locking is needed in here, because ->can_attach() will 3843 * check this value once in the beginning of the process, and then carry 3844 * on with stale data. This means that changes to this value will only 3845 * affect task migrations starting after the change. 3846 */ 3847 memcg->move_charge_at_immigrate = val; 3848 return 0; 3849 } 3850 #else 3851 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css, 3852 struct cftype *cft, u64 val) 3853 { 3854 return -ENOSYS; 3855 } 3856 #endif 3857 3858 #ifdef CONFIG_NUMA 3859 3860 #define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE)) 3861 #define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON)) 3862 #define LRU_ALL ((1 << NR_LRU_LISTS) - 1) 3863 3864 static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg, 3865 int nid, unsigned int lru_mask, bool tree) 3866 { 3867 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid)); 3868 unsigned long nr = 0; 3869 enum lru_list lru; 3870 3871 VM_BUG_ON((unsigned)nid >= nr_node_ids); 3872 3873 for_each_lru(lru) { 3874 if (!(BIT(lru) & lru_mask)) 3875 continue; 3876 if (tree) 3877 nr += lruvec_page_state(lruvec, NR_LRU_BASE + lru); 3878 else 3879 nr += lruvec_page_state_local(lruvec, NR_LRU_BASE + lru); 3880 } 3881 return nr; 3882 } 3883 3884 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg, 3885 unsigned int lru_mask, 3886 bool tree) 3887 { 3888 unsigned long nr = 0; 3889 enum lru_list lru; 3890 3891 for_each_lru(lru) { 3892 if (!(BIT(lru) & lru_mask)) 3893 continue; 3894 if (tree) 3895 nr += memcg_page_state(memcg, NR_LRU_BASE + lru); 3896 else 3897 nr += memcg_page_state_local(memcg, NR_LRU_BASE + lru); 3898 } 3899 return nr; 3900 } 3901 3902 static int memcg_numa_stat_show(struct seq_file *m, void *v) 3903 { 3904 struct numa_stat { 3905 const char *name; 3906 unsigned int lru_mask; 3907 }; 3908 3909 static const struct numa_stat stats[] = { 3910 { "total", LRU_ALL }, 3911 { "file", LRU_ALL_FILE }, 3912 { "anon", LRU_ALL_ANON }, 3913 { "unevictable", BIT(LRU_UNEVICTABLE) }, 3914 }; 3915 const struct numa_stat *stat; 3916 int nid; 3917 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 3918 3919 cgroup_rstat_flush(memcg->css.cgroup); 3920 3921 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) { 3922 seq_printf(m, "%s=%lu", stat->name, 3923 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask, 3924 false)); 3925 for_each_node_state(nid, N_MEMORY) 3926 seq_printf(m, " N%d=%lu", nid, 3927 mem_cgroup_node_nr_lru_pages(memcg, nid, 3928 stat->lru_mask, false)); 3929 seq_putc(m, '\n'); 3930 } 3931 3932 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) { 3933 3934 seq_printf(m, "hierarchical_%s=%lu", stat->name, 3935 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask, 3936 true)); 3937 for_each_node_state(nid, N_MEMORY) 3938 seq_printf(m, " N%d=%lu", nid, 3939 mem_cgroup_node_nr_lru_pages(memcg, nid, 3940 stat->lru_mask, true)); 3941 seq_putc(m, '\n'); 3942 } 3943 3944 return 0; 3945 } 3946 #endif /* CONFIG_NUMA */ 3947 3948 static const unsigned int memcg1_stats[] = { 3949 NR_FILE_PAGES, 3950 NR_ANON_MAPPED, 3951 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3952 NR_ANON_THPS, 3953 #endif 3954 NR_SHMEM, 3955 NR_FILE_MAPPED, 3956 NR_FILE_DIRTY, 3957 NR_WRITEBACK, 3958 MEMCG_SWAP, 3959 }; 3960 3961 static const char *const memcg1_stat_names[] = { 3962 "cache", 3963 "rss", 3964 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3965 "rss_huge", 3966 #endif 3967 "shmem", 3968 "mapped_file", 3969 "dirty", 3970 "writeback", 3971 "swap", 3972 }; 3973 3974 /* Universal VM events cgroup1 shows, original sort order */ 3975 static const unsigned int memcg1_events[] = { 3976 PGPGIN, 3977 PGPGOUT, 3978 PGFAULT, 3979 PGMAJFAULT, 3980 }; 3981 3982 static int memcg_stat_show(struct seq_file *m, void *v) 3983 { 3984 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 3985 unsigned long memory, memsw; 3986 struct mem_cgroup *mi; 3987 unsigned int i; 3988 3989 BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats)); 3990 3991 cgroup_rstat_flush(memcg->css.cgroup); 3992 3993 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) { 3994 unsigned long nr; 3995 3996 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account()) 3997 continue; 3998 nr = memcg_page_state_local(memcg, memcg1_stats[i]); 3999 seq_printf(m, "%s %lu\n", memcg1_stat_names[i], nr * PAGE_SIZE); 4000 } 4001 4002 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++) 4003 seq_printf(m, "%s %lu\n", vm_event_name(memcg1_events[i]), 4004 memcg_events_local(memcg, memcg1_events[i])); 4005 4006 for (i = 0; i < NR_LRU_LISTS; i++) 4007 seq_printf(m, "%s %lu\n", lru_list_name(i), 4008 memcg_page_state_local(memcg, NR_LRU_BASE + i) * 4009 PAGE_SIZE); 4010 4011 /* Hierarchical information */ 4012 memory = memsw = PAGE_COUNTER_MAX; 4013 for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) { 4014 memory = min(memory, READ_ONCE(mi->memory.max)); 4015 memsw = min(memsw, READ_ONCE(mi->memsw.max)); 4016 } 4017 seq_printf(m, "hierarchical_memory_limit %llu\n", 4018 (u64)memory * PAGE_SIZE); 4019 if (do_memsw_account()) 4020 seq_printf(m, "hierarchical_memsw_limit %llu\n", 4021 (u64)memsw * PAGE_SIZE); 4022 4023 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) { 4024 unsigned long nr; 4025 4026 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account()) 4027 continue; 4028 nr = memcg_page_state(memcg, memcg1_stats[i]); 4029 seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i], 4030 (u64)nr * PAGE_SIZE); 4031 } 4032 4033 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++) 4034 seq_printf(m, "total_%s %llu\n", 4035 vm_event_name(memcg1_events[i]), 4036 (u64)memcg_events(memcg, memcg1_events[i])); 4037 4038 for (i = 0; i < NR_LRU_LISTS; i++) 4039 seq_printf(m, "total_%s %llu\n", lru_list_name(i), 4040 (u64)memcg_page_state(memcg, NR_LRU_BASE + i) * 4041 PAGE_SIZE); 4042 4043 #ifdef CONFIG_DEBUG_VM 4044 { 4045 pg_data_t *pgdat; 4046 struct mem_cgroup_per_node *mz; 4047 unsigned long anon_cost = 0; 4048 unsigned long file_cost = 0; 4049 4050 for_each_online_pgdat(pgdat) { 4051 mz = memcg->nodeinfo[pgdat->node_id]; 4052 4053 anon_cost += mz->lruvec.anon_cost; 4054 file_cost += mz->lruvec.file_cost; 4055 } 4056 seq_printf(m, "anon_cost %lu\n", anon_cost); 4057 seq_printf(m, "file_cost %lu\n", file_cost); 4058 } 4059 #endif 4060 4061 return 0; 4062 } 4063 4064 static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css, 4065 struct cftype *cft) 4066 { 4067 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4068 4069 return mem_cgroup_swappiness(memcg); 4070 } 4071 4072 static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css, 4073 struct cftype *cft, u64 val) 4074 { 4075 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4076 4077 if (val > 200) 4078 return -EINVAL; 4079 4080 if (!mem_cgroup_is_root(memcg)) 4081 memcg->swappiness = val; 4082 else 4083 vm_swappiness = val; 4084 4085 return 0; 4086 } 4087 4088 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap) 4089 { 4090 struct mem_cgroup_threshold_ary *t; 4091 unsigned long usage; 4092 int i; 4093 4094 rcu_read_lock(); 4095 if (!swap) 4096 t = rcu_dereference(memcg->thresholds.primary); 4097 else 4098 t = rcu_dereference(memcg->memsw_thresholds.primary); 4099 4100 if (!t) 4101 goto unlock; 4102 4103 usage = mem_cgroup_usage(memcg, swap); 4104 4105 /* 4106 * current_threshold points to threshold just below or equal to usage. 4107 * If it's not true, a threshold was crossed after last 4108 * call of __mem_cgroup_threshold(). 4109 */ 4110 i = t->current_threshold; 4111 4112 /* 4113 * Iterate backward over array of thresholds starting from 4114 * current_threshold and check if a threshold is crossed. 4115 * If none of thresholds below usage is crossed, we read 4116 * only one element of the array here. 4117 */ 4118 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--) 4119 eventfd_signal(t->entries[i].eventfd, 1); 4120 4121 /* i = current_threshold + 1 */ 4122 i++; 4123 4124 /* 4125 * Iterate forward over array of thresholds starting from 4126 * current_threshold+1 and check if a threshold is crossed. 4127 * If none of thresholds above usage is crossed, we read 4128 * only one element of the array here. 4129 */ 4130 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++) 4131 eventfd_signal(t->entries[i].eventfd, 1); 4132 4133 /* Update current_threshold */ 4134 t->current_threshold = i - 1; 4135 unlock: 4136 rcu_read_unlock(); 4137 } 4138 4139 static void mem_cgroup_threshold(struct mem_cgroup *memcg) 4140 { 4141 while (memcg) { 4142 __mem_cgroup_threshold(memcg, false); 4143 if (do_memsw_account()) 4144 __mem_cgroup_threshold(memcg, true); 4145 4146 memcg = parent_mem_cgroup(memcg); 4147 } 4148 } 4149 4150 static int compare_thresholds(const void *a, const void *b) 4151 { 4152 const struct mem_cgroup_threshold *_a = a; 4153 const struct mem_cgroup_threshold *_b = b; 4154 4155 if (_a->threshold > _b->threshold) 4156 return 1; 4157 4158 if (_a->threshold < _b->threshold) 4159 return -1; 4160 4161 return 0; 4162 } 4163 4164 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg) 4165 { 4166 struct mem_cgroup_eventfd_list *ev; 4167 4168 spin_lock(&memcg_oom_lock); 4169 4170 list_for_each_entry(ev, &memcg->oom_notify, list) 4171 eventfd_signal(ev->eventfd, 1); 4172 4173 spin_unlock(&memcg_oom_lock); 4174 return 0; 4175 } 4176 4177 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg) 4178 { 4179 struct mem_cgroup *iter; 4180 4181 for_each_mem_cgroup_tree(iter, memcg) 4182 mem_cgroup_oom_notify_cb(iter); 4183 } 4184 4185 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg, 4186 struct eventfd_ctx *eventfd, const char *args, enum res_type type) 4187 { 4188 struct mem_cgroup_thresholds *thresholds; 4189 struct mem_cgroup_threshold_ary *new; 4190 unsigned long threshold; 4191 unsigned long usage; 4192 int i, size, ret; 4193 4194 ret = page_counter_memparse(args, "-1", &threshold); 4195 if (ret) 4196 return ret; 4197 4198 mutex_lock(&memcg->thresholds_lock); 4199 4200 if (type == _MEM) { 4201 thresholds = &memcg->thresholds; 4202 usage = mem_cgroup_usage(memcg, false); 4203 } else if (type == _MEMSWAP) { 4204 thresholds = &memcg->memsw_thresholds; 4205 usage = mem_cgroup_usage(memcg, true); 4206 } else 4207 BUG(); 4208 4209 /* Check if a threshold crossed before adding a new one */ 4210 if (thresholds->primary) 4211 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 4212 4213 size = thresholds->primary ? thresholds->primary->size + 1 : 1; 4214 4215 /* Allocate memory for new array of thresholds */ 4216 new = kmalloc(struct_size(new, entries, size), GFP_KERNEL); 4217 if (!new) { 4218 ret = -ENOMEM; 4219 goto unlock; 4220 } 4221 new->size = size; 4222 4223 /* Copy thresholds (if any) to new array */ 4224 if (thresholds->primary) 4225 memcpy(new->entries, thresholds->primary->entries, 4226 flex_array_size(new, entries, size - 1)); 4227 4228 /* Add new threshold */ 4229 new->entries[size - 1].eventfd = eventfd; 4230 new->entries[size - 1].threshold = threshold; 4231 4232 /* Sort thresholds. Registering of new threshold isn't time-critical */ 4233 sort(new->entries, size, sizeof(*new->entries), 4234 compare_thresholds, NULL); 4235 4236 /* Find current threshold */ 4237 new->current_threshold = -1; 4238 for (i = 0; i < size; i++) { 4239 if (new->entries[i].threshold <= usage) { 4240 /* 4241 * new->current_threshold will not be used until 4242 * rcu_assign_pointer(), so it's safe to increment 4243 * it here. 4244 */ 4245 ++new->current_threshold; 4246 } else 4247 break; 4248 } 4249 4250 /* Free old spare buffer and save old primary buffer as spare */ 4251 kfree(thresholds->spare); 4252 thresholds->spare = thresholds->primary; 4253 4254 rcu_assign_pointer(thresholds->primary, new); 4255 4256 /* To be sure that nobody uses thresholds */ 4257 synchronize_rcu(); 4258 4259 unlock: 4260 mutex_unlock(&memcg->thresholds_lock); 4261 4262 return ret; 4263 } 4264 4265 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg, 4266 struct eventfd_ctx *eventfd, const char *args) 4267 { 4268 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM); 4269 } 4270 4271 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg, 4272 struct eventfd_ctx *eventfd, const char *args) 4273 { 4274 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP); 4275 } 4276 4277 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, 4278 struct eventfd_ctx *eventfd, enum res_type type) 4279 { 4280 struct mem_cgroup_thresholds *thresholds; 4281 struct mem_cgroup_threshold_ary *new; 4282 unsigned long usage; 4283 int i, j, size, entries; 4284 4285 mutex_lock(&memcg->thresholds_lock); 4286 4287 if (type == _MEM) { 4288 thresholds = &memcg->thresholds; 4289 usage = mem_cgroup_usage(memcg, false); 4290 } else if (type == _MEMSWAP) { 4291 thresholds = &memcg->memsw_thresholds; 4292 usage = mem_cgroup_usage(memcg, true); 4293 } else 4294 BUG(); 4295 4296 if (!thresholds->primary) 4297 goto unlock; 4298 4299 /* Check if a threshold crossed before removing */ 4300 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 4301 4302 /* Calculate new number of threshold */ 4303 size = entries = 0; 4304 for (i = 0; i < thresholds->primary->size; i++) { 4305 if (thresholds->primary->entries[i].eventfd != eventfd) 4306 size++; 4307 else 4308 entries++; 4309 } 4310 4311 new = thresholds->spare; 4312 4313 /* If no items related to eventfd have been cleared, nothing to do */ 4314 if (!entries) 4315 goto unlock; 4316 4317 /* Set thresholds array to NULL if we don't have thresholds */ 4318 if (!size) { 4319 kfree(new); 4320 new = NULL; 4321 goto swap_buffers; 4322 } 4323 4324 new->size = size; 4325 4326 /* Copy thresholds and find current threshold */ 4327 new->current_threshold = -1; 4328 for (i = 0, j = 0; i < thresholds->primary->size; i++) { 4329 if (thresholds->primary->entries[i].eventfd == eventfd) 4330 continue; 4331 4332 new->entries[j] = thresholds->primary->entries[i]; 4333 if (new->entries[j].threshold <= usage) { 4334 /* 4335 * new->current_threshold will not be used 4336 * until rcu_assign_pointer(), so it's safe to increment 4337 * it here. 4338 */ 4339 ++new->current_threshold; 4340 } 4341 j++; 4342 } 4343 4344 swap_buffers: 4345 /* Swap primary and spare array */ 4346 thresholds->spare = thresholds->primary; 4347 4348 rcu_assign_pointer(thresholds->primary, new); 4349 4350 /* To be sure that nobody uses thresholds */ 4351 synchronize_rcu(); 4352 4353 /* If all events are unregistered, free the spare array */ 4354 if (!new) { 4355 kfree(thresholds->spare); 4356 thresholds->spare = NULL; 4357 } 4358 unlock: 4359 mutex_unlock(&memcg->thresholds_lock); 4360 } 4361 4362 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, 4363 struct eventfd_ctx *eventfd) 4364 { 4365 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM); 4366 } 4367 4368 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg, 4369 struct eventfd_ctx *eventfd) 4370 { 4371 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP); 4372 } 4373 4374 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg, 4375 struct eventfd_ctx *eventfd, const char *args) 4376 { 4377 struct mem_cgroup_eventfd_list *event; 4378 4379 event = kmalloc(sizeof(*event), GFP_KERNEL); 4380 if (!event) 4381 return -ENOMEM; 4382 4383 spin_lock(&memcg_oom_lock); 4384 4385 event->eventfd = eventfd; 4386 list_add(&event->list, &memcg->oom_notify); 4387 4388 /* already in OOM ? */ 4389 if (memcg->under_oom) 4390 eventfd_signal(eventfd, 1); 4391 spin_unlock(&memcg_oom_lock); 4392 4393 return 0; 4394 } 4395 4396 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg, 4397 struct eventfd_ctx *eventfd) 4398 { 4399 struct mem_cgroup_eventfd_list *ev, *tmp; 4400 4401 spin_lock(&memcg_oom_lock); 4402 4403 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) { 4404 if (ev->eventfd == eventfd) { 4405 list_del(&ev->list); 4406 kfree(ev); 4407 } 4408 } 4409 4410 spin_unlock(&memcg_oom_lock); 4411 } 4412 4413 static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v) 4414 { 4415 struct mem_cgroup *memcg = mem_cgroup_from_seq(sf); 4416 4417 seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable); 4418 seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom); 4419 seq_printf(sf, "oom_kill %lu\n", 4420 atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL])); 4421 return 0; 4422 } 4423 4424 static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css, 4425 struct cftype *cft, u64 val) 4426 { 4427 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4428 4429 /* cannot set to root cgroup and only 0 and 1 are allowed */ 4430 if (mem_cgroup_is_root(memcg) || !((val == 0) || (val == 1))) 4431 return -EINVAL; 4432 4433 memcg->oom_kill_disable = val; 4434 if (!val) 4435 memcg_oom_recover(memcg); 4436 4437 return 0; 4438 } 4439 4440 #ifdef CONFIG_CGROUP_WRITEBACK 4441 4442 #include <trace/events/writeback.h> 4443 4444 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp) 4445 { 4446 return wb_domain_init(&memcg->cgwb_domain, gfp); 4447 } 4448 4449 static void memcg_wb_domain_exit(struct mem_cgroup *memcg) 4450 { 4451 wb_domain_exit(&memcg->cgwb_domain); 4452 } 4453 4454 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg) 4455 { 4456 wb_domain_size_changed(&memcg->cgwb_domain); 4457 } 4458 4459 struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb) 4460 { 4461 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); 4462 4463 if (!memcg->css.parent) 4464 return NULL; 4465 4466 return &memcg->cgwb_domain; 4467 } 4468 4469 /** 4470 * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg 4471 * @wb: bdi_writeback in question 4472 * @pfilepages: out parameter for number of file pages 4473 * @pheadroom: out parameter for number of allocatable pages according to memcg 4474 * @pdirty: out parameter for number of dirty pages 4475 * @pwriteback: out parameter for number of pages under writeback 4476 * 4477 * Determine the numbers of file, headroom, dirty, and writeback pages in 4478 * @wb's memcg. File, dirty and writeback are self-explanatory. Headroom 4479 * is a bit more involved. 4480 * 4481 * A memcg's headroom is "min(max, high) - used". In the hierarchy, the 4482 * headroom is calculated as the lowest headroom of itself and the 4483 * ancestors. Note that this doesn't consider the actual amount of 4484 * available memory in the system. The caller should further cap 4485 * *@pheadroom accordingly. 4486 */ 4487 void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages, 4488 unsigned long *pheadroom, unsigned long *pdirty, 4489 unsigned long *pwriteback) 4490 { 4491 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); 4492 struct mem_cgroup *parent; 4493 4494 cgroup_rstat_flush_irqsafe(memcg->css.cgroup); 4495 4496 *pdirty = memcg_page_state(memcg, NR_FILE_DIRTY); 4497 *pwriteback = memcg_page_state(memcg, NR_WRITEBACK); 4498 *pfilepages = memcg_page_state(memcg, NR_INACTIVE_FILE) + 4499 memcg_page_state(memcg, NR_ACTIVE_FILE); 4500 4501 *pheadroom = PAGE_COUNTER_MAX; 4502 while ((parent = parent_mem_cgroup(memcg))) { 4503 unsigned long ceiling = min(READ_ONCE(memcg->memory.max), 4504 READ_ONCE(memcg->memory.high)); 4505 unsigned long used = page_counter_read(&memcg->memory); 4506 4507 *pheadroom = min(*pheadroom, ceiling - min(ceiling, used)); 4508 memcg = parent; 4509 } 4510 } 4511 4512 /* 4513 * Foreign dirty flushing 4514 * 4515 * There's an inherent mismatch between memcg and writeback. The former 4516 * tracks ownership per-page while the latter per-inode. This was a 4517 * deliberate design decision because honoring per-page ownership in the 4518 * writeback path is complicated, may lead to higher CPU and IO overheads 4519 * and deemed unnecessary given that write-sharing an inode across 4520 * different cgroups isn't a common use-case. 4521 * 4522 * Combined with inode majority-writer ownership switching, this works well 4523 * enough in most cases but there are some pathological cases. For 4524 * example, let's say there are two cgroups A and B which keep writing to 4525 * different but confined parts of the same inode. B owns the inode and 4526 * A's memory is limited far below B's. A's dirty ratio can rise enough to 4527 * trigger balance_dirty_pages() sleeps but B's can be low enough to avoid 4528 * triggering background writeback. A will be slowed down without a way to 4529 * make writeback of the dirty pages happen. 4530 * 4531 * Conditions like the above can lead to a cgroup getting repeatedly and 4532 * severely throttled after making some progress after each 4533 * dirty_expire_interval while the underlying IO device is almost 4534 * completely idle. 4535 * 4536 * Solving this problem completely requires matching the ownership tracking 4537 * granularities between memcg and writeback in either direction. However, 4538 * the more egregious behaviors can be avoided by simply remembering the 4539 * most recent foreign dirtying events and initiating remote flushes on 4540 * them when local writeback isn't enough to keep the memory clean enough. 4541 * 4542 * The following two functions implement such mechanism. When a foreign 4543 * page - a page whose memcg and writeback ownerships don't match - is 4544 * dirtied, mem_cgroup_track_foreign_dirty() records the inode owning 4545 * bdi_writeback on the page owning memcg. When balance_dirty_pages() 4546 * decides that the memcg needs to sleep due to high dirty ratio, it calls 4547 * mem_cgroup_flush_foreign() which queues writeback on the recorded 4548 * foreign bdi_writebacks which haven't expired. Both the numbers of 4549 * recorded bdi_writebacks and concurrent in-flight foreign writebacks are 4550 * limited to MEMCG_CGWB_FRN_CNT. 4551 * 4552 * The mechanism only remembers IDs and doesn't hold any object references. 4553 * As being wrong occasionally doesn't matter, updates and accesses to the 4554 * records are lockless and racy. 4555 */ 4556 void mem_cgroup_track_foreign_dirty_slowpath(struct folio *folio, 4557 struct bdi_writeback *wb) 4558 { 4559 struct mem_cgroup *memcg = folio_memcg(folio); 4560 struct memcg_cgwb_frn *frn; 4561 u64 now = get_jiffies_64(); 4562 u64 oldest_at = now; 4563 int oldest = -1; 4564 int i; 4565 4566 trace_track_foreign_dirty(folio, wb); 4567 4568 /* 4569 * Pick the slot to use. If there is already a slot for @wb, keep 4570 * using it. If not replace the oldest one which isn't being 4571 * written out. 4572 */ 4573 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) { 4574 frn = &memcg->cgwb_frn[i]; 4575 if (frn->bdi_id == wb->bdi->id && 4576 frn->memcg_id == wb->memcg_css->id) 4577 break; 4578 if (time_before64(frn->at, oldest_at) && 4579 atomic_read(&frn->done.cnt) == 1) { 4580 oldest = i; 4581 oldest_at = frn->at; 4582 } 4583 } 4584 4585 if (i < MEMCG_CGWB_FRN_CNT) { 4586 /* 4587 * Re-using an existing one. Update timestamp lazily to 4588 * avoid making the cacheline hot. We want them to be 4589 * reasonably up-to-date and significantly shorter than 4590 * dirty_expire_interval as that's what expires the record. 4591 * Use the shorter of 1s and dirty_expire_interval / 8. 4592 */ 4593 unsigned long update_intv = 4594 min_t(unsigned long, HZ, 4595 msecs_to_jiffies(dirty_expire_interval * 10) / 8); 4596 4597 if (time_before64(frn->at, now - update_intv)) 4598 frn->at = now; 4599 } else if (oldest >= 0) { 4600 /* replace the oldest free one */ 4601 frn = &memcg->cgwb_frn[oldest]; 4602 frn->bdi_id = wb->bdi->id; 4603 frn->memcg_id = wb->memcg_css->id; 4604 frn->at = now; 4605 } 4606 } 4607 4608 /* issue foreign writeback flushes for recorded foreign dirtying events */ 4609 void mem_cgroup_flush_foreign(struct bdi_writeback *wb) 4610 { 4611 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); 4612 unsigned long intv = msecs_to_jiffies(dirty_expire_interval * 10); 4613 u64 now = jiffies_64; 4614 int i; 4615 4616 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) { 4617 struct memcg_cgwb_frn *frn = &memcg->cgwb_frn[i]; 4618 4619 /* 4620 * If the record is older than dirty_expire_interval, 4621 * writeback on it has already started. No need to kick it 4622 * off again. Also, don't start a new one if there's 4623 * already one in flight. 4624 */ 4625 if (time_after64(frn->at, now - intv) && 4626 atomic_read(&frn->done.cnt) == 1) { 4627 frn->at = 0; 4628 trace_flush_foreign(wb, frn->bdi_id, frn->memcg_id); 4629 cgroup_writeback_by_id(frn->bdi_id, frn->memcg_id, 4630 WB_REASON_FOREIGN_FLUSH, 4631 &frn->done); 4632 } 4633 } 4634 } 4635 4636 #else /* CONFIG_CGROUP_WRITEBACK */ 4637 4638 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp) 4639 { 4640 return 0; 4641 } 4642 4643 static void memcg_wb_domain_exit(struct mem_cgroup *memcg) 4644 { 4645 } 4646 4647 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg) 4648 { 4649 } 4650 4651 #endif /* CONFIG_CGROUP_WRITEBACK */ 4652 4653 /* 4654 * DO NOT USE IN NEW FILES. 4655 * 4656 * "cgroup.event_control" implementation. 4657 * 4658 * This is way over-engineered. It tries to support fully configurable 4659 * events for each user. Such level of flexibility is completely 4660 * unnecessary especially in the light of the planned unified hierarchy. 4661 * 4662 * Please deprecate this and replace with something simpler if at all 4663 * possible. 4664 */ 4665 4666 /* 4667 * Unregister event and free resources. 4668 * 4669 * Gets called from workqueue. 4670 */ 4671 static void memcg_event_remove(struct work_struct *work) 4672 { 4673 struct mem_cgroup_event *event = 4674 container_of(work, struct mem_cgroup_event, remove); 4675 struct mem_cgroup *memcg = event->memcg; 4676 4677 remove_wait_queue(event->wqh, &event->wait); 4678 4679 event->unregister_event(memcg, event->eventfd); 4680 4681 /* Notify userspace the event is going away. */ 4682 eventfd_signal(event->eventfd, 1); 4683 4684 eventfd_ctx_put(event->eventfd); 4685 kfree(event); 4686 css_put(&memcg->css); 4687 } 4688 4689 /* 4690 * Gets called on EPOLLHUP on eventfd when user closes it. 4691 * 4692 * Called with wqh->lock held and interrupts disabled. 4693 */ 4694 static int memcg_event_wake(wait_queue_entry_t *wait, unsigned mode, 4695 int sync, void *key) 4696 { 4697 struct mem_cgroup_event *event = 4698 container_of(wait, struct mem_cgroup_event, wait); 4699 struct mem_cgroup *memcg = event->memcg; 4700 __poll_t flags = key_to_poll(key); 4701 4702 if (flags & EPOLLHUP) { 4703 /* 4704 * If the event has been detached at cgroup removal, we 4705 * can simply return knowing the other side will cleanup 4706 * for us. 4707 * 4708 * We can't race against event freeing since the other 4709 * side will require wqh->lock via remove_wait_queue(), 4710 * which we hold. 4711 */ 4712 spin_lock(&memcg->event_list_lock); 4713 if (!list_empty(&event->list)) { 4714 list_del_init(&event->list); 4715 /* 4716 * We are in atomic context, but cgroup_event_remove() 4717 * may sleep, so we have to call it in workqueue. 4718 */ 4719 schedule_work(&event->remove); 4720 } 4721 spin_unlock(&memcg->event_list_lock); 4722 } 4723 4724 return 0; 4725 } 4726 4727 static void memcg_event_ptable_queue_proc(struct file *file, 4728 wait_queue_head_t *wqh, poll_table *pt) 4729 { 4730 struct mem_cgroup_event *event = 4731 container_of(pt, struct mem_cgroup_event, pt); 4732 4733 event->wqh = wqh; 4734 add_wait_queue(wqh, &event->wait); 4735 } 4736 4737 /* 4738 * DO NOT USE IN NEW FILES. 4739 * 4740 * Parse input and register new cgroup event handler. 4741 * 4742 * Input must be in format '<event_fd> <control_fd> <args>'. 4743 * Interpretation of args is defined by control file implementation. 4744 */ 4745 static ssize_t memcg_write_event_control(struct kernfs_open_file *of, 4746 char *buf, size_t nbytes, loff_t off) 4747 { 4748 struct cgroup_subsys_state *css = of_css(of); 4749 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4750 struct mem_cgroup_event *event; 4751 struct cgroup_subsys_state *cfile_css; 4752 unsigned int efd, cfd; 4753 struct fd efile; 4754 struct fd cfile; 4755 const char *name; 4756 char *endp; 4757 int ret; 4758 4759 buf = strstrip(buf); 4760 4761 efd = simple_strtoul(buf, &endp, 10); 4762 if (*endp != ' ') 4763 return -EINVAL; 4764 buf = endp + 1; 4765 4766 cfd = simple_strtoul(buf, &endp, 10); 4767 if ((*endp != ' ') && (*endp != '\0')) 4768 return -EINVAL; 4769 buf = endp + 1; 4770 4771 event = kzalloc(sizeof(*event), GFP_KERNEL); 4772 if (!event) 4773 return -ENOMEM; 4774 4775 event->memcg = memcg; 4776 INIT_LIST_HEAD(&event->list); 4777 init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc); 4778 init_waitqueue_func_entry(&event->wait, memcg_event_wake); 4779 INIT_WORK(&event->remove, memcg_event_remove); 4780 4781 efile = fdget(efd); 4782 if (!efile.file) { 4783 ret = -EBADF; 4784 goto out_kfree; 4785 } 4786 4787 event->eventfd = eventfd_ctx_fileget(efile.file); 4788 if (IS_ERR(event->eventfd)) { 4789 ret = PTR_ERR(event->eventfd); 4790 goto out_put_efile; 4791 } 4792 4793 cfile = fdget(cfd); 4794 if (!cfile.file) { 4795 ret = -EBADF; 4796 goto out_put_eventfd; 4797 } 4798 4799 /* the process need read permission on control file */ 4800 /* AV: shouldn't we check that it's been opened for read instead? */ 4801 ret = file_permission(cfile.file, MAY_READ); 4802 if (ret < 0) 4803 goto out_put_cfile; 4804 4805 /* 4806 * Determine the event callbacks and set them in @event. This used 4807 * to be done via struct cftype but cgroup core no longer knows 4808 * about these events. The following is crude but the whole thing 4809 * is for compatibility anyway. 4810 * 4811 * DO NOT ADD NEW FILES. 4812 */ 4813 name = cfile.file->f_path.dentry->d_name.name; 4814 4815 if (!strcmp(name, "memory.usage_in_bytes")) { 4816 event->register_event = mem_cgroup_usage_register_event; 4817 event->unregister_event = mem_cgroup_usage_unregister_event; 4818 } else if (!strcmp(name, "memory.oom_control")) { 4819 event->register_event = mem_cgroup_oom_register_event; 4820 event->unregister_event = mem_cgroup_oom_unregister_event; 4821 } else if (!strcmp(name, "memory.pressure_level")) { 4822 event->register_event = vmpressure_register_event; 4823 event->unregister_event = vmpressure_unregister_event; 4824 } else if (!strcmp(name, "memory.memsw.usage_in_bytes")) { 4825 event->register_event = memsw_cgroup_usage_register_event; 4826 event->unregister_event = memsw_cgroup_usage_unregister_event; 4827 } else { 4828 ret = -EINVAL; 4829 goto out_put_cfile; 4830 } 4831 4832 /* 4833 * Verify @cfile should belong to @css. Also, remaining events are 4834 * automatically removed on cgroup destruction but the removal is 4835 * asynchronous, so take an extra ref on @css. 4836 */ 4837 cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent, 4838 &memory_cgrp_subsys); 4839 ret = -EINVAL; 4840 if (IS_ERR(cfile_css)) 4841 goto out_put_cfile; 4842 if (cfile_css != css) { 4843 css_put(cfile_css); 4844 goto out_put_cfile; 4845 } 4846 4847 ret = event->register_event(memcg, event->eventfd, buf); 4848 if (ret) 4849 goto out_put_css; 4850 4851 vfs_poll(efile.file, &event->pt); 4852 4853 spin_lock_irq(&memcg->event_list_lock); 4854 list_add(&event->list, &memcg->event_list); 4855 spin_unlock_irq(&memcg->event_list_lock); 4856 4857 fdput(cfile); 4858 fdput(efile); 4859 4860 return nbytes; 4861 4862 out_put_css: 4863 css_put(css); 4864 out_put_cfile: 4865 fdput(cfile); 4866 out_put_eventfd: 4867 eventfd_ctx_put(event->eventfd); 4868 out_put_efile: 4869 fdput(efile); 4870 out_kfree: 4871 kfree(event); 4872 4873 return ret; 4874 } 4875 4876 static struct cftype mem_cgroup_legacy_files[] = { 4877 { 4878 .name = "usage_in_bytes", 4879 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE), 4880 .read_u64 = mem_cgroup_read_u64, 4881 }, 4882 { 4883 .name = "max_usage_in_bytes", 4884 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE), 4885 .write = mem_cgroup_reset, 4886 .read_u64 = mem_cgroup_read_u64, 4887 }, 4888 { 4889 .name = "limit_in_bytes", 4890 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT), 4891 .write = mem_cgroup_write, 4892 .read_u64 = mem_cgroup_read_u64, 4893 }, 4894 { 4895 .name = "soft_limit_in_bytes", 4896 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT), 4897 .write = mem_cgroup_write, 4898 .read_u64 = mem_cgroup_read_u64, 4899 }, 4900 { 4901 .name = "failcnt", 4902 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT), 4903 .write = mem_cgroup_reset, 4904 .read_u64 = mem_cgroup_read_u64, 4905 }, 4906 { 4907 .name = "stat", 4908 .seq_show = memcg_stat_show, 4909 }, 4910 { 4911 .name = "force_empty", 4912 .write = mem_cgroup_force_empty_write, 4913 }, 4914 { 4915 .name = "use_hierarchy", 4916 .write_u64 = mem_cgroup_hierarchy_write, 4917 .read_u64 = mem_cgroup_hierarchy_read, 4918 }, 4919 { 4920 .name = "cgroup.event_control", /* XXX: for compat */ 4921 .write = memcg_write_event_control, 4922 .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE, 4923 }, 4924 { 4925 .name = "swappiness", 4926 .read_u64 = mem_cgroup_swappiness_read, 4927 .write_u64 = mem_cgroup_swappiness_write, 4928 }, 4929 { 4930 .name = "move_charge_at_immigrate", 4931 .read_u64 = mem_cgroup_move_charge_read, 4932 .write_u64 = mem_cgroup_move_charge_write, 4933 }, 4934 { 4935 .name = "oom_control", 4936 .seq_show = mem_cgroup_oom_control_read, 4937 .write_u64 = mem_cgroup_oom_control_write, 4938 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL), 4939 }, 4940 { 4941 .name = "pressure_level", 4942 }, 4943 #ifdef CONFIG_NUMA 4944 { 4945 .name = "numa_stat", 4946 .seq_show = memcg_numa_stat_show, 4947 }, 4948 #endif 4949 { 4950 .name = "kmem.limit_in_bytes", 4951 .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT), 4952 .write = mem_cgroup_write, 4953 .read_u64 = mem_cgroup_read_u64, 4954 }, 4955 { 4956 .name = "kmem.usage_in_bytes", 4957 .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE), 4958 .read_u64 = mem_cgroup_read_u64, 4959 }, 4960 { 4961 .name = "kmem.failcnt", 4962 .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT), 4963 .write = mem_cgroup_reset, 4964 .read_u64 = mem_cgroup_read_u64, 4965 }, 4966 { 4967 .name = "kmem.max_usage_in_bytes", 4968 .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE), 4969 .write = mem_cgroup_reset, 4970 .read_u64 = mem_cgroup_read_u64, 4971 }, 4972 #if defined(CONFIG_MEMCG_KMEM) && \ 4973 (defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)) 4974 { 4975 .name = "kmem.slabinfo", 4976 .seq_show = memcg_slab_show, 4977 }, 4978 #endif 4979 { 4980 .name = "kmem.tcp.limit_in_bytes", 4981 .private = MEMFILE_PRIVATE(_TCP, RES_LIMIT), 4982 .write = mem_cgroup_write, 4983 .read_u64 = mem_cgroup_read_u64, 4984 }, 4985 { 4986 .name = "kmem.tcp.usage_in_bytes", 4987 .private = MEMFILE_PRIVATE(_TCP, RES_USAGE), 4988 .read_u64 = mem_cgroup_read_u64, 4989 }, 4990 { 4991 .name = "kmem.tcp.failcnt", 4992 .private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT), 4993 .write = mem_cgroup_reset, 4994 .read_u64 = mem_cgroup_read_u64, 4995 }, 4996 { 4997 .name = "kmem.tcp.max_usage_in_bytes", 4998 .private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE), 4999 .write = mem_cgroup_reset, 5000 .read_u64 = mem_cgroup_read_u64, 5001 }, 5002 { }, /* terminate */ 5003 }; 5004 5005 /* 5006 * Private memory cgroup IDR 5007 * 5008 * Swap-out records and page cache shadow entries need to store memcg 5009 * references in constrained space, so we maintain an ID space that is 5010 * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of 5011 * memory-controlled cgroups to 64k. 5012 * 5013 * However, there usually are many references to the offline CSS after 5014 * the cgroup has been destroyed, such as page cache or reclaimable 5015 * slab objects, that don't need to hang on to the ID. We want to keep 5016 * those dead CSS from occupying IDs, or we might quickly exhaust the 5017 * relatively small ID space and prevent the creation of new cgroups 5018 * even when there are much fewer than 64k cgroups - possibly none. 5019 * 5020 * Maintain a private 16-bit ID space for memcg, and allow the ID to 5021 * be freed and recycled when it's no longer needed, which is usually 5022 * when the CSS is offlined. 5023 * 5024 * The only exception to that are records of swapped out tmpfs/shmem 5025 * pages that need to be attributed to live ancestors on swapin. But 5026 * those references are manageable from userspace. 5027 */ 5028 5029 static DEFINE_IDR(mem_cgroup_idr); 5030 5031 static void mem_cgroup_id_remove(struct mem_cgroup *memcg) 5032 { 5033 if (memcg->id.id > 0) { 5034 idr_remove(&mem_cgroup_idr, memcg->id.id); 5035 memcg->id.id = 0; 5036 } 5037 } 5038 5039 static void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg, 5040 unsigned int n) 5041 { 5042 refcount_add(n, &memcg->id.ref); 5043 } 5044 5045 static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n) 5046 { 5047 if (refcount_sub_and_test(n, &memcg->id.ref)) { 5048 mem_cgroup_id_remove(memcg); 5049 5050 /* Memcg ID pins CSS */ 5051 css_put(&memcg->css); 5052 } 5053 } 5054 5055 static inline void mem_cgroup_id_put(struct mem_cgroup *memcg) 5056 { 5057 mem_cgroup_id_put_many(memcg, 1); 5058 } 5059 5060 /** 5061 * mem_cgroup_from_id - look up a memcg from a memcg id 5062 * @id: the memcg id to look up 5063 * 5064 * Caller must hold rcu_read_lock(). 5065 */ 5066 struct mem_cgroup *mem_cgroup_from_id(unsigned short id) 5067 { 5068 WARN_ON_ONCE(!rcu_read_lock_held()); 5069 return idr_find(&mem_cgroup_idr, id); 5070 } 5071 5072 static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) 5073 { 5074 struct mem_cgroup_per_node *pn; 5075 int tmp = node; 5076 /* 5077 * This routine is called against possible nodes. 5078 * But it's BUG to call kmalloc() against offline node. 5079 * 5080 * TODO: this routine can waste much memory for nodes which will 5081 * never be onlined. It's better to use memory hotplug callback 5082 * function. 5083 */ 5084 if (!node_state(node, N_NORMAL_MEMORY)) 5085 tmp = -1; 5086 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp); 5087 if (!pn) 5088 return 1; 5089 5090 pn->lruvec_stats_percpu = alloc_percpu_gfp(struct lruvec_stats_percpu, 5091 GFP_KERNEL_ACCOUNT); 5092 if (!pn->lruvec_stats_percpu) { 5093 kfree(pn); 5094 return 1; 5095 } 5096 5097 lruvec_init(&pn->lruvec); 5098 pn->usage_in_excess = 0; 5099 pn->on_tree = false; 5100 pn->memcg = memcg; 5101 5102 memcg->nodeinfo[node] = pn; 5103 return 0; 5104 } 5105 5106 static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) 5107 { 5108 struct mem_cgroup_per_node *pn = memcg->nodeinfo[node]; 5109 5110 if (!pn) 5111 return; 5112 5113 free_percpu(pn->lruvec_stats_percpu); 5114 kfree(pn); 5115 } 5116 5117 static void __mem_cgroup_free(struct mem_cgroup *memcg) 5118 { 5119 int node; 5120 5121 for_each_node(node) 5122 free_mem_cgroup_per_node_info(memcg, node); 5123 free_percpu(memcg->vmstats_percpu); 5124 kfree(memcg); 5125 } 5126 5127 static void mem_cgroup_free(struct mem_cgroup *memcg) 5128 { 5129 memcg_wb_domain_exit(memcg); 5130 __mem_cgroup_free(memcg); 5131 } 5132 5133 static struct mem_cgroup *mem_cgroup_alloc(void) 5134 { 5135 struct mem_cgroup *memcg; 5136 unsigned int size; 5137 int node; 5138 int __maybe_unused i; 5139 long error = -ENOMEM; 5140 5141 size = sizeof(struct mem_cgroup); 5142 size += nr_node_ids * sizeof(struct mem_cgroup_per_node *); 5143 5144 memcg = kzalloc(size, GFP_KERNEL); 5145 if (!memcg) 5146 return ERR_PTR(error); 5147 5148 memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL, 5149 1, MEM_CGROUP_ID_MAX, 5150 GFP_KERNEL); 5151 if (memcg->id.id < 0) { 5152 error = memcg->id.id; 5153 goto fail; 5154 } 5155 5156 memcg->vmstats_percpu = alloc_percpu_gfp(struct memcg_vmstats_percpu, 5157 GFP_KERNEL_ACCOUNT); 5158 if (!memcg->vmstats_percpu) 5159 goto fail; 5160 5161 for_each_node(node) 5162 if (alloc_mem_cgroup_per_node_info(memcg, node)) 5163 goto fail; 5164 5165 if (memcg_wb_domain_init(memcg, GFP_KERNEL)) 5166 goto fail; 5167 5168 INIT_WORK(&memcg->high_work, high_work_func); 5169 INIT_LIST_HEAD(&memcg->oom_notify); 5170 mutex_init(&memcg->thresholds_lock); 5171 spin_lock_init(&memcg->move_lock); 5172 vmpressure_init(&memcg->vmpressure); 5173 INIT_LIST_HEAD(&memcg->event_list); 5174 spin_lock_init(&memcg->event_list_lock); 5175 memcg->socket_pressure = jiffies; 5176 #ifdef CONFIG_MEMCG_KMEM 5177 memcg->kmemcg_id = -1; 5178 INIT_LIST_HEAD(&memcg->objcg_list); 5179 #endif 5180 #ifdef CONFIG_CGROUP_WRITEBACK 5181 INIT_LIST_HEAD(&memcg->cgwb_list); 5182 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) 5183 memcg->cgwb_frn[i].done = 5184 __WB_COMPLETION_INIT(&memcg_cgwb_frn_waitq); 5185 #endif 5186 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5187 spin_lock_init(&memcg->deferred_split_queue.split_queue_lock); 5188 INIT_LIST_HEAD(&memcg->deferred_split_queue.split_queue); 5189 memcg->deferred_split_queue.split_queue_len = 0; 5190 #endif 5191 idr_replace(&mem_cgroup_idr, memcg, memcg->id.id); 5192 return memcg; 5193 fail: 5194 mem_cgroup_id_remove(memcg); 5195 __mem_cgroup_free(memcg); 5196 return ERR_PTR(error); 5197 } 5198 5199 static struct cgroup_subsys_state * __ref 5200 mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 5201 { 5202 struct mem_cgroup *parent = mem_cgroup_from_css(parent_css); 5203 struct mem_cgroup *memcg, *old_memcg; 5204 long error = -ENOMEM; 5205 5206 old_memcg = set_active_memcg(parent); 5207 memcg = mem_cgroup_alloc(); 5208 set_active_memcg(old_memcg); 5209 if (IS_ERR(memcg)) 5210 return ERR_CAST(memcg); 5211 5212 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX); 5213 memcg->soft_limit = PAGE_COUNTER_MAX; 5214 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX); 5215 if (parent) { 5216 memcg->swappiness = mem_cgroup_swappiness(parent); 5217 memcg->oom_kill_disable = parent->oom_kill_disable; 5218 5219 page_counter_init(&memcg->memory, &parent->memory); 5220 page_counter_init(&memcg->swap, &parent->swap); 5221 page_counter_init(&memcg->kmem, &parent->kmem); 5222 page_counter_init(&memcg->tcpmem, &parent->tcpmem); 5223 } else { 5224 page_counter_init(&memcg->memory, NULL); 5225 page_counter_init(&memcg->swap, NULL); 5226 page_counter_init(&memcg->kmem, NULL); 5227 page_counter_init(&memcg->tcpmem, NULL); 5228 5229 root_mem_cgroup = memcg; 5230 return &memcg->css; 5231 } 5232 5233 /* The following stuff does not apply to the root */ 5234 error = memcg_online_kmem(memcg); 5235 if (error) 5236 goto fail; 5237 5238 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket) 5239 static_branch_inc(&memcg_sockets_enabled_key); 5240 5241 return &memcg->css; 5242 fail: 5243 mem_cgroup_id_remove(memcg); 5244 mem_cgroup_free(memcg); 5245 return ERR_PTR(error); 5246 } 5247 5248 static int mem_cgroup_css_online(struct cgroup_subsys_state *css) 5249 { 5250 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5251 5252 /* 5253 * A memcg must be visible for expand_shrinker_info() 5254 * by the time the maps are allocated. So, we allocate maps 5255 * here, when for_each_mem_cgroup() can't skip it. 5256 */ 5257 if (alloc_shrinker_info(memcg)) { 5258 mem_cgroup_id_remove(memcg); 5259 return -ENOMEM; 5260 } 5261 5262 /* Online state pins memcg ID, memcg ID pins CSS */ 5263 refcount_set(&memcg->id.ref, 1); 5264 css_get(css); 5265 5266 if (unlikely(mem_cgroup_is_root(memcg))) 5267 queue_delayed_work(system_unbound_wq, &stats_flush_dwork, 5268 2UL*HZ); 5269 return 0; 5270 } 5271 5272 static void mem_cgroup_css_offline(struct cgroup_subsys_state *css) 5273 { 5274 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5275 struct mem_cgroup_event *event, *tmp; 5276 5277 /* 5278 * Unregister events and notify userspace. 5279 * Notify userspace about cgroup removing only after rmdir of cgroup 5280 * directory to avoid race between userspace and kernelspace. 5281 */ 5282 spin_lock_irq(&memcg->event_list_lock); 5283 list_for_each_entry_safe(event, tmp, &memcg->event_list, list) { 5284 list_del_init(&event->list); 5285 schedule_work(&event->remove); 5286 } 5287 spin_unlock_irq(&memcg->event_list_lock); 5288 5289 page_counter_set_min(&memcg->memory, 0); 5290 page_counter_set_low(&memcg->memory, 0); 5291 5292 memcg_offline_kmem(memcg); 5293 reparent_shrinker_deferred(memcg); 5294 wb_memcg_offline(memcg); 5295 5296 drain_all_stock(memcg); 5297 5298 mem_cgroup_id_put(memcg); 5299 } 5300 5301 static void mem_cgroup_css_released(struct cgroup_subsys_state *css) 5302 { 5303 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5304 5305 invalidate_reclaim_iterators(memcg); 5306 } 5307 5308 static void mem_cgroup_css_free(struct cgroup_subsys_state *css) 5309 { 5310 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5311 int __maybe_unused i; 5312 5313 #ifdef CONFIG_CGROUP_WRITEBACK 5314 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) 5315 wb_wait_for_completion(&memcg->cgwb_frn[i].done); 5316 #endif 5317 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket) 5318 static_branch_dec(&memcg_sockets_enabled_key); 5319 5320 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active) 5321 static_branch_dec(&memcg_sockets_enabled_key); 5322 5323 vmpressure_cleanup(&memcg->vmpressure); 5324 cancel_work_sync(&memcg->high_work); 5325 mem_cgroup_remove_from_trees(memcg); 5326 free_shrinker_info(memcg); 5327 memcg_free_kmem(memcg); 5328 mem_cgroup_free(memcg); 5329 } 5330 5331 /** 5332 * mem_cgroup_css_reset - reset the states of a mem_cgroup 5333 * @css: the target css 5334 * 5335 * Reset the states of the mem_cgroup associated with @css. This is 5336 * invoked when the userland requests disabling on the default hierarchy 5337 * but the memcg is pinned through dependency. The memcg should stop 5338 * applying policies and should revert to the vanilla state as it may be 5339 * made visible again. 5340 * 5341 * The current implementation only resets the essential configurations. 5342 * This needs to be expanded to cover all the visible parts. 5343 */ 5344 static void mem_cgroup_css_reset(struct cgroup_subsys_state *css) 5345 { 5346 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5347 5348 page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX); 5349 page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX); 5350 page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX); 5351 page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX); 5352 page_counter_set_min(&memcg->memory, 0); 5353 page_counter_set_low(&memcg->memory, 0); 5354 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX); 5355 memcg->soft_limit = PAGE_COUNTER_MAX; 5356 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX); 5357 memcg_wb_domain_size_changed(memcg); 5358 } 5359 5360 void mem_cgroup_flush_stats(void) 5361 { 5362 if (!spin_trylock(&stats_flush_lock)) 5363 return; 5364 5365 cgroup_rstat_flush_irqsafe(root_mem_cgroup->css.cgroup); 5366 spin_unlock(&stats_flush_lock); 5367 } 5368 5369 static void flush_memcg_stats_dwork(struct work_struct *w) 5370 { 5371 mem_cgroup_flush_stats(); 5372 queue_delayed_work(system_unbound_wq, &stats_flush_dwork, 2UL*HZ); 5373 } 5374 5375 static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu) 5376 { 5377 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5378 struct mem_cgroup *parent = parent_mem_cgroup(memcg); 5379 struct memcg_vmstats_percpu *statc; 5380 long delta, v; 5381 int i, nid; 5382 5383 statc = per_cpu_ptr(memcg->vmstats_percpu, cpu); 5384 5385 for (i = 0; i < MEMCG_NR_STAT; i++) { 5386 /* 5387 * Collect the aggregated propagation counts of groups 5388 * below us. We're in a per-cpu loop here and this is 5389 * a global counter, so the first cycle will get them. 5390 */ 5391 delta = memcg->vmstats.state_pending[i]; 5392 if (delta) 5393 memcg->vmstats.state_pending[i] = 0; 5394 5395 /* Add CPU changes on this level since the last flush */ 5396 v = READ_ONCE(statc->state[i]); 5397 if (v != statc->state_prev[i]) { 5398 delta += v - statc->state_prev[i]; 5399 statc->state_prev[i] = v; 5400 } 5401 5402 if (!delta) 5403 continue; 5404 5405 /* Aggregate counts on this level and propagate upwards */ 5406 memcg->vmstats.state[i] += delta; 5407 if (parent) 5408 parent->vmstats.state_pending[i] += delta; 5409 } 5410 5411 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) { 5412 delta = memcg->vmstats.events_pending[i]; 5413 if (delta) 5414 memcg->vmstats.events_pending[i] = 0; 5415 5416 v = READ_ONCE(statc->events[i]); 5417 if (v != statc->events_prev[i]) { 5418 delta += v - statc->events_prev[i]; 5419 statc->events_prev[i] = v; 5420 } 5421 5422 if (!delta) 5423 continue; 5424 5425 memcg->vmstats.events[i] += delta; 5426 if (parent) 5427 parent->vmstats.events_pending[i] += delta; 5428 } 5429 5430 for_each_node_state(nid, N_MEMORY) { 5431 struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid]; 5432 struct mem_cgroup_per_node *ppn = NULL; 5433 struct lruvec_stats_percpu *lstatc; 5434 5435 if (parent) 5436 ppn = parent->nodeinfo[nid]; 5437 5438 lstatc = per_cpu_ptr(pn->lruvec_stats_percpu, cpu); 5439 5440 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 5441 delta = pn->lruvec_stats.state_pending[i]; 5442 if (delta) 5443 pn->lruvec_stats.state_pending[i] = 0; 5444 5445 v = READ_ONCE(lstatc->state[i]); 5446 if (v != lstatc->state_prev[i]) { 5447 delta += v - lstatc->state_prev[i]; 5448 lstatc->state_prev[i] = v; 5449 } 5450 5451 if (!delta) 5452 continue; 5453 5454 pn->lruvec_stats.state[i] += delta; 5455 if (ppn) 5456 ppn->lruvec_stats.state_pending[i] += delta; 5457 } 5458 } 5459 } 5460 5461 #ifdef CONFIG_MMU 5462 /* Handlers for move charge at task migration. */ 5463 static int mem_cgroup_do_precharge(unsigned long count) 5464 { 5465 int ret; 5466 5467 /* Try a single bulk charge without reclaim first, kswapd may wake */ 5468 ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count); 5469 if (!ret) { 5470 mc.precharge += count; 5471 return ret; 5472 } 5473 5474 /* Try charges one by one with reclaim, but do not retry */ 5475 while (count--) { 5476 ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1); 5477 if (ret) 5478 return ret; 5479 mc.precharge++; 5480 cond_resched(); 5481 } 5482 return 0; 5483 } 5484 5485 union mc_target { 5486 struct page *page; 5487 swp_entry_t ent; 5488 }; 5489 5490 enum mc_target_type { 5491 MC_TARGET_NONE = 0, 5492 MC_TARGET_PAGE, 5493 MC_TARGET_SWAP, 5494 MC_TARGET_DEVICE, 5495 }; 5496 5497 static struct page *mc_handle_present_pte(struct vm_area_struct *vma, 5498 unsigned long addr, pte_t ptent) 5499 { 5500 struct page *page = vm_normal_page(vma, addr, ptent); 5501 5502 if (!page || !page_mapped(page)) 5503 return NULL; 5504 if (PageAnon(page)) { 5505 if (!(mc.flags & MOVE_ANON)) 5506 return NULL; 5507 } else { 5508 if (!(mc.flags & MOVE_FILE)) 5509 return NULL; 5510 } 5511 if (!get_page_unless_zero(page)) 5512 return NULL; 5513 5514 return page; 5515 } 5516 5517 #if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE) 5518 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma, 5519 pte_t ptent, swp_entry_t *entry) 5520 { 5521 struct page *page = NULL; 5522 swp_entry_t ent = pte_to_swp_entry(ptent); 5523 5524 if (!(mc.flags & MOVE_ANON)) 5525 return NULL; 5526 5527 /* 5528 * Handle MEMORY_DEVICE_PRIVATE which are ZONE_DEVICE page belonging to 5529 * a device and because they are not accessible by CPU they are store 5530 * as special swap entry in the CPU page table. 5531 */ 5532 if (is_device_private_entry(ent)) { 5533 page = pfn_swap_entry_to_page(ent); 5534 /* 5535 * MEMORY_DEVICE_PRIVATE means ZONE_DEVICE page and which have 5536 * a refcount of 1 when free (unlike normal page) 5537 */ 5538 if (!page_ref_add_unless(page, 1, 1)) 5539 return NULL; 5540 return page; 5541 } 5542 5543 if (non_swap_entry(ent)) 5544 return NULL; 5545 5546 /* 5547 * Because lookup_swap_cache() updates some statistics counter, 5548 * we call find_get_page() with swapper_space directly. 5549 */ 5550 page = find_get_page(swap_address_space(ent), swp_offset(ent)); 5551 entry->val = ent.val; 5552 5553 return page; 5554 } 5555 #else 5556 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma, 5557 pte_t ptent, swp_entry_t *entry) 5558 { 5559 return NULL; 5560 } 5561 #endif 5562 5563 static struct page *mc_handle_file_pte(struct vm_area_struct *vma, 5564 unsigned long addr, pte_t ptent, swp_entry_t *entry) 5565 { 5566 if (!vma->vm_file) /* anonymous vma */ 5567 return NULL; 5568 if (!(mc.flags & MOVE_FILE)) 5569 return NULL; 5570 5571 /* page is moved even if it's not RSS of this task(page-faulted). */ 5572 /* shmem/tmpfs may report page out on swap: account for that too. */ 5573 return find_get_incore_page(vma->vm_file->f_mapping, 5574 linear_page_index(vma, addr)); 5575 } 5576 5577 /** 5578 * mem_cgroup_move_account - move account of the page 5579 * @page: the page 5580 * @compound: charge the page as compound or small page 5581 * @from: mem_cgroup which the page is moved from. 5582 * @to: mem_cgroup which the page is moved to. @from != @to. 5583 * 5584 * The caller must make sure the page is not on LRU (isolate_page() is useful.) 5585 * 5586 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge" 5587 * from old cgroup. 5588 */ 5589 static int mem_cgroup_move_account(struct page *page, 5590 bool compound, 5591 struct mem_cgroup *from, 5592 struct mem_cgroup *to) 5593 { 5594 struct folio *folio = page_folio(page); 5595 struct lruvec *from_vec, *to_vec; 5596 struct pglist_data *pgdat; 5597 unsigned int nr_pages = compound ? folio_nr_pages(folio) : 1; 5598 int nid, ret; 5599 5600 VM_BUG_ON(from == to); 5601 VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); 5602 VM_BUG_ON(compound && !folio_test_multi(folio)); 5603 5604 /* 5605 * Prevent mem_cgroup_migrate() from looking at 5606 * page's memory cgroup of its source page while we change it. 5607 */ 5608 ret = -EBUSY; 5609 if (!folio_trylock(folio)) 5610 goto out; 5611 5612 ret = -EINVAL; 5613 if (folio_memcg(folio) != from) 5614 goto out_unlock; 5615 5616 pgdat = folio_pgdat(folio); 5617 from_vec = mem_cgroup_lruvec(from, pgdat); 5618 to_vec = mem_cgroup_lruvec(to, pgdat); 5619 5620 folio_memcg_lock(folio); 5621 5622 if (folio_test_anon(folio)) { 5623 if (folio_mapped(folio)) { 5624 __mod_lruvec_state(from_vec, NR_ANON_MAPPED, -nr_pages); 5625 __mod_lruvec_state(to_vec, NR_ANON_MAPPED, nr_pages); 5626 if (folio_test_transhuge(folio)) { 5627 __mod_lruvec_state(from_vec, NR_ANON_THPS, 5628 -nr_pages); 5629 __mod_lruvec_state(to_vec, NR_ANON_THPS, 5630 nr_pages); 5631 } 5632 } 5633 } else { 5634 __mod_lruvec_state(from_vec, NR_FILE_PAGES, -nr_pages); 5635 __mod_lruvec_state(to_vec, NR_FILE_PAGES, nr_pages); 5636 5637 if (folio_test_swapbacked(folio)) { 5638 __mod_lruvec_state(from_vec, NR_SHMEM, -nr_pages); 5639 __mod_lruvec_state(to_vec, NR_SHMEM, nr_pages); 5640 } 5641 5642 if (folio_mapped(folio)) { 5643 __mod_lruvec_state(from_vec, NR_FILE_MAPPED, -nr_pages); 5644 __mod_lruvec_state(to_vec, NR_FILE_MAPPED, nr_pages); 5645 } 5646 5647 if (folio_test_dirty(folio)) { 5648 struct address_space *mapping = folio_mapping(folio); 5649 5650 if (mapping_can_writeback(mapping)) { 5651 __mod_lruvec_state(from_vec, NR_FILE_DIRTY, 5652 -nr_pages); 5653 __mod_lruvec_state(to_vec, NR_FILE_DIRTY, 5654 nr_pages); 5655 } 5656 } 5657 } 5658 5659 if (folio_test_writeback(folio)) { 5660 __mod_lruvec_state(from_vec, NR_WRITEBACK, -nr_pages); 5661 __mod_lruvec_state(to_vec, NR_WRITEBACK, nr_pages); 5662 } 5663 5664 /* 5665 * All state has been migrated, let's switch to the new memcg. 5666 * 5667 * It is safe to change page's memcg here because the page 5668 * is referenced, charged, isolated, and locked: we can't race 5669 * with (un)charging, migration, LRU putback, or anything else 5670 * that would rely on a stable page's memory cgroup. 5671 * 5672 * Note that lock_page_memcg is a memcg lock, not a page lock, 5673 * to save space. As soon as we switch page's memory cgroup to a 5674 * new memcg that isn't locked, the above state can change 5675 * concurrently again. Make sure we're truly done with it. 5676 */ 5677 smp_mb(); 5678 5679 css_get(&to->css); 5680 css_put(&from->css); 5681 5682 folio->memcg_data = (unsigned long)to; 5683 5684 __folio_memcg_unlock(from); 5685 5686 ret = 0; 5687 nid = folio_nid(folio); 5688 5689 local_irq_disable(); 5690 mem_cgroup_charge_statistics(to, nr_pages); 5691 memcg_check_events(to, nid); 5692 mem_cgroup_charge_statistics(from, -nr_pages); 5693 memcg_check_events(from, nid); 5694 local_irq_enable(); 5695 out_unlock: 5696 folio_unlock(folio); 5697 out: 5698 return ret; 5699 } 5700 5701 /** 5702 * get_mctgt_type - get target type of moving charge 5703 * @vma: the vma the pte to be checked belongs 5704 * @addr: the address corresponding to the pte to be checked 5705 * @ptent: the pte to be checked 5706 * @target: the pointer the target page or swap ent will be stored(can be NULL) 5707 * 5708 * Returns 5709 * 0(MC_TARGET_NONE): if the pte is not a target for move charge. 5710 * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for 5711 * move charge. if @target is not NULL, the page is stored in target->page 5712 * with extra refcnt got(Callers should handle it). 5713 * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a 5714 * target for charge migration. if @target is not NULL, the entry is stored 5715 * in target->ent. 5716 * 3(MC_TARGET_DEVICE): like MC_TARGET_PAGE but page is MEMORY_DEVICE_PRIVATE 5717 * (so ZONE_DEVICE page and thus not on the lru). 5718 * For now we such page is charge like a regular page would be as for all 5719 * intent and purposes it is just special memory taking the place of a 5720 * regular page. 5721 * 5722 * See Documentations/vm/hmm.txt and include/linux/hmm.h 5723 * 5724 * Called with pte lock held. 5725 */ 5726 5727 static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma, 5728 unsigned long addr, pte_t ptent, union mc_target *target) 5729 { 5730 struct page *page = NULL; 5731 enum mc_target_type ret = MC_TARGET_NONE; 5732 swp_entry_t ent = { .val = 0 }; 5733 5734 if (pte_present(ptent)) 5735 page = mc_handle_present_pte(vma, addr, ptent); 5736 else if (is_swap_pte(ptent)) 5737 page = mc_handle_swap_pte(vma, ptent, &ent); 5738 else if (pte_none(ptent)) 5739 page = mc_handle_file_pte(vma, addr, ptent, &ent); 5740 5741 if (!page && !ent.val) 5742 return ret; 5743 if (page) { 5744 /* 5745 * Do only loose check w/o serialization. 5746 * mem_cgroup_move_account() checks the page is valid or 5747 * not under LRU exclusion. 5748 */ 5749 if (page_memcg(page) == mc.from) { 5750 ret = MC_TARGET_PAGE; 5751 if (is_device_private_page(page)) 5752 ret = MC_TARGET_DEVICE; 5753 if (target) 5754 target->page = page; 5755 } 5756 if (!ret || !target) 5757 put_page(page); 5758 } 5759 /* 5760 * There is a swap entry and a page doesn't exist or isn't charged. 5761 * But we cannot move a tail-page in a THP. 5762 */ 5763 if (ent.val && !ret && (!page || !PageTransCompound(page)) && 5764 mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) { 5765 ret = MC_TARGET_SWAP; 5766 if (target) 5767 target->ent = ent; 5768 } 5769 return ret; 5770 } 5771 5772 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5773 /* 5774 * We don't consider PMD mapped swapping or file mapped pages because THP does 5775 * not support them for now. 5776 * Caller should make sure that pmd_trans_huge(pmd) is true. 5777 */ 5778 static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma, 5779 unsigned long addr, pmd_t pmd, union mc_target *target) 5780 { 5781 struct page *page = NULL; 5782 enum mc_target_type ret = MC_TARGET_NONE; 5783 5784 if (unlikely(is_swap_pmd(pmd))) { 5785 VM_BUG_ON(thp_migration_supported() && 5786 !is_pmd_migration_entry(pmd)); 5787 return ret; 5788 } 5789 page = pmd_page(pmd); 5790 VM_BUG_ON_PAGE(!page || !PageHead(page), page); 5791 if (!(mc.flags & MOVE_ANON)) 5792 return ret; 5793 if (page_memcg(page) == mc.from) { 5794 ret = MC_TARGET_PAGE; 5795 if (target) { 5796 get_page(page); 5797 target->page = page; 5798 } 5799 } 5800 return ret; 5801 } 5802 #else 5803 static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma, 5804 unsigned long addr, pmd_t pmd, union mc_target *target) 5805 { 5806 return MC_TARGET_NONE; 5807 } 5808 #endif 5809 5810 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd, 5811 unsigned long addr, unsigned long end, 5812 struct mm_walk *walk) 5813 { 5814 struct vm_area_struct *vma = walk->vma; 5815 pte_t *pte; 5816 spinlock_t *ptl; 5817 5818 ptl = pmd_trans_huge_lock(pmd, vma); 5819 if (ptl) { 5820 /* 5821 * Note their can not be MC_TARGET_DEVICE for now as we do not 5822 * support transparent huge page with MEMORY_DEVICE_PRIVATE but 5823 * this might change. 5824 */ 5825 if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE) 5826 mc.precharge += HPAGE_PMD_NR; 5827 spin_unlock(ptl); 5828 return 0; 5829 } 5830 5831 if (pmd_trans_unstable(pmd)) 5832 return 0; 5833 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 5834 for (; addr != end; pte++, addr += PAGE_SIZE) 5835 if (get_mctgt_type(vma, addr, *pte, NULL)) 5836 mc.precharge++; /* increment precharge temporarily */ 5837 pte_unmap_unlock(pte - 1, ptl); 5838 cond_resched(); 5839 5840 return 0; 5841 } 5842 5843 static const struct mm_walk_ops precharge_walk_ops = { 5844 .pmd_entry = mem_cgroup_count_precharge_pte_range, 5845 }; 5846 5847 static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm) 5848 { 5849 unsigned long precharge; 5850 5851 mmap_read_lock(mm); 5852 walk_page_range(mm, 0, mm->highest_vm_end, &precharge_walk_ops, NULL); 5853 mmap_read_unlock(mm); 5854 5855 precharge = mc.precharge; 5856 mc.precharge = 0; 5857 5858 return precharge; 5859 } 5860 5861 static int mem_cgroup_precharge_mc(struct mm_struct *mm) 5862 { 5863 unsigned long precharge = mem_cgroup_count_precharge(mm); 5864 5865 VM_BUG_ON(mc.moving_task); 5866 mc.moving_task = current; 5867 return mem_cgroup_do_precharge(precharge); 5868 } 5869 5870 /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */ 5871 static void __mem_cgroup_clear_mc(void) 5872 { 5873 struct mem_cgroup *from = mc.from; 5874 struct mem_cgroup *to = mc.to; 5875 5876 /* we must uncharge all the leftover precharges from mc.to */ 5877 if (mc.precharge) { 5878 cancel_charge(mc.to, mc.precharge); 5879 mc.precharge = 0; 5880 } 5881 /* 5882 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so 5883 * we must uncharge here. 5884 */ 5885 if (mc.moved_charge) { 5886 cancel_charge(mc.from, mc.moved_charge); 5887 mc.moved_charge = 0; 5888 } 5889 /* we must fixup refcnts and charges */ 5890 if (mc.moved_swap) { 5891 /* uncharge swap account from the old cgroup */ 5892 if (!mem_cgroup_is_root(mc.from)) 5893 page_counter_uncharge(&mc.from->memsw, mc.moved_swap); 5894 5895 mem_cgroup_id_put_many(mc.from, mc.moved_swap); 5896 5897 /* 5898 * we charged both to->memory and to->memsw, so we 5899 * should uncharge to->memory. 5900 */ 5901 if (!mem_cgroup_is_root(mc.to)) 5902 page_counter_uncharge(&mc.to->memory, mc.moved_swap); 5903 5904 mc.moved_swap = 0; 5905 } 5906 memcg_oom_recover(from); 5907 memcg_oom_recover(to); 5908 wake_up_all(&mc.waitq); 5909 } 5910 5911 static void mem_cgroup_clear_mc(void) 5912 { 5913 struct mm_struct *mm = mc.mm; 5914 5915 /* 5916 * we must clear moving_task before waking up waiters at the end of 5917 * task migration. 5918 */ 5919 mc.moving_task = NULL; 5920 __mem_cgroup_clear_mc(); 5921 spin_lock(&mc.lock); 5922 mc.from = NULL; 5923 mc.to = NULL; 5924 mc.mm = NULL; 5925 spin_unlock(&mc.lock); 5926 5927 mmput(mm); 5928 } 5929 5930 static int mem_cgroup_can_attach(struct cgroup_taskset *tset) 5931 { 5932 struct cgroup_subsys_state *css; 5933 struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */ 5934 struct mem_cgroup *from; 5935 struct task_struct *leader, *p; 5936 struct mm_struct *mm; 5937 unsigned long move_flags; 5938 int ret = 0; 5939 5940 /* charge immigration isn't supported on the default hierarchy */ 5941 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 5942 return 0; 5943 5944 /* 5945 * Multi-process migrations only happen on the default hierarchy 5946 * where charge immigration is not used. Perform charge 5947 * immigration if @tset contains a leader and whine if there are 5948 * multiple. 5949 */ 5950 p = NULL; 5951 cgroup_taskset_for_each_leader(leader, css, tset) { 5952 WARN_ON_ONCE(p); 5953 p = leader; 5954 memcg = mem_cgroup_from_css(css); 5955 } 5956 if (!p) 5957 return 0; 5958 5959 /* 5960 * We are now committed to this value whatever it is. Changes in this 5961 * tunable will only affect upcoming migrations, not the current one. 5962 * So we need to save it, and keep it going. 5963 */ 5964 move_flags = READ_ONCE(memcg->move_charge_at_immigrate); 5965 if (!move_flags) 5966 return 0; 5967 5968 from = mem_cgroup_from_task(p); 5969 5970 VM_BUG_ON(from == memcg); 5971 5972 mm = get_task_mm(p); 5973 if (!mm) 5974 return 0; 5975 /* We move charges only when we move a owner of the mm */ 5976 if (mm->owner == p) { 5977 VM_BUG_ON(mc.from); 5978 VM_BUG_ON(mc.to); 5979 VM_BUG_ON(mc.precharge); 5980 VM_BUG_ON(mc.moved_charge); 5981 VM_BUG_ON(mc.moved_swap); 5982 5983 spin_lock(&mc.lock); 5984 mc.mm = mm; 5985 mc.from = from; 5986 mc.to = memcg; 5987 mc.flags = move_flags; 5988 spin_unlock(&mc.lock); 5989 /* We set mc.moving_task later */ 5990 5991 ret = mem_cgroup_precharge_mc(mm); 5992 if (ret) 5993 mem_cgroup_clear_mc(); 5994 } else { 5995 mmput(mm); 5996 } 5997 return ret; 5998 } 5999 6000 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset) 6001 { 6002 if (mc.to) 6003 mem_cgroup_clear_mc(); 6004 } 6005 6006 static int mem_cgroup_move_charge_pte_range(pmd_t *pmd, 6007 unsigned long addr, unsigned long end, 6008 struct mm_walk *walk) 6009 { 6010 int ret = 0; 6011 struct vm_area_struct *vma = walk->vma; 6012 pte_t *pte; 6013 spinlock_t *ptl; 6014 enum mc_target_type target_type; 6015 union mc_target target; 6016 struct page *page; 6017 6018 ptl = pmd_trans_huge_lock(pmd, vma); 6019 if (ptl) { 6020 if (mc.precharge < HPAGE_PMD_NR) { 6021 spin_unlock(ptl); 6022 return 0; 6023 } 6024 target_type = get_mctgt_type_thp(vma, addr, *pmd, &target); 6025 if (target_type == MC_TARGET_PAGE) { 6026 page = target.page; 6027 if (!isolate_lru_page(page)) { 6028 if (!mem_cgroup_move_account(page, true, 6029 mc.from, mc.to)) { 6030 mc.precharge -= HPAGE_PMD_NR; 6031 mc.moved_charge += HPAGE_PMD_NR; 6032 } 6033 putback_lru_page(page); 6034 } 6035 put_page(page); 6036 } else if (target_type == MC_TARGET_DEVICE) { 6037 page = target.page; 6038 if (!mem_cgroup_move_account(page, true, 6039 mc.from, mc.to)) { 6040 mc.precharge -= HPAGE_PMD_NR; 6041 mc.moved_charge += HPAGE_PMD_NR; 6042 } 6043 put_page(page); 6044 } 6045 spin_unlock(ptl); 6046 return 0; 6047 } 6048 6049 if (pmd_trans_unstable(pmd)) 6050 return 0; 6051 retry: 6052 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 6053 for (; addr != end; addr += PAGE_SIZE) { 6054 pte_t ptent = *(pte++); 6055 bool device = false; 6056 swp_entry_t ent; 6057 6058 if (!mc.precharge) 6059 break; 6060 6061 switch (get_mctgt_type(vma, addr, ptent, &target)) { 6062 case MC_TARGET_DEVICE: 6063 device = true; 6064 fallthrough; 6065 case MC_TARGET_PAGE: 6066 page = target.page; 6067 /* 6068 * We can have a part of the split pmd here. Moving it 6069 * can be done but it would be too convoluted so simply 6070 * ignore such a partial THP and keep it in original 6071 * memcg. There should be somebody mapping the head. 6072 */ 6073 if (PageTransCompound(page)) 6074 goto put; 6075 if (!device && isolate_lru_page(page)) 6076 goto put; 6077 if (!mem_cgroup_move_account(page, false, 6078 mc.from, mc.to)) { 6079 mc.precharge--; 6080 /* we uncharge from mc.from later. */ 6081 mc.moved_charge++; 6082 } 6083 if (!device) 6084 putback_lru_page(page); 6085 put: /* get_mctgt_type() gets the page */ 6086 put_page(page); 6087 break; 6088 case MC_TARGET_SWAP: 6089 ent = target.ent; 6090 if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) { 6091 mc.precharge--; 6092 mem_cgroup_id_get_many(mc.to, 1); 6093 /* we fixup other refcnts and charges later. */ 6094 mc.moved_swap++; 6095 } 6096 break; 6097 default: 6098 break; 6099 } 6100 } 6101 pte_unmap_unlock(pte - 1, ptl); 6102 cond_resched(); 6103 6104 if (addr != end) { 6105 /* 6106 * We have consumed all precharges we got in can_attach(). 6107 * We try charge one by one, but don't do any additional 6108 * charges to mc.to if we have failed in charge once in attach() 6109 * phase. 6110 */ 6111 ret = mem_cgroup_do_precharge(1); 6112 if (!ret) 6113 goto retry; 6114 } 6115 6116 return ret; 6117 } 6118 6119 static const struct mm_walk_ops charge_walk_ops = { 6120 .pmd_entry = mem_cgroup_move_charge_pte_range, 6121 }; 6122 6123 static void mem_cgroup_move_charge(void) 6124 { 6125 lru_add_drain_all(); 6126 /* 6127 * Signal lock_page_memcg() to take the memcg's move_lock 6128 * while we're moving its pages to another memcg. Then wait 6129 * for already started RCU-only updates to finish. 6130 */ 6131 atomic_inc(&mc.from->moving_account); 6132 synchronize_rcu(); 6133 retry: 6134 if (unlikely(!mmap_read_trylock(mc.mm))) { 6135 /* 6136 * Someone who are holding the mmap_lock might be waiting in 6137 * waitq. So we cancel all extra charges, wake up all waiters, 6138 * and retry. Because we cancel precharges, we might not be able 6139 * to move enough charges, but moving charge is a best-effort 6140 * feature anyway, so it wouldn't be a big problem. 6141 */ 6142 __mem_cgroup_clear_mc(); 6143 cond_resched(); 6144 goto retry; 6145 } 6146 /* 6147 * When we have consumed all precharges and failed in doing 6148 * additional charge, the page walk just aborts. 6149 */ 6150 walk_page_range(mc.mm, 0, mc.mm->highest_vm_end, &charge_walk_ops, 6151 NULL); 6152 6153 mmap_read_unlock(mc.mm); 6154 atomic_dec(&mc.from->moving_account); 6155 } 6156 6157 static void mem_cgroup_move_task(void) 6158 { 6159 if (mc.to) { 6160 mem_cgroup_move_charge(); 6161 mem_cgroup_clear_mc(); 6162 } 6163 } 6164 #else /* !CONFIG_MMU */ 6165 static int mem_cgroup_can_attach(struct cgroup_taskset *tset) 6166 { 6167 return 0; 6168 } 6169 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset) 6170 { 6171 } 6172 static void mem_cgroup_move_task(void) 6173 { 6174 } 6175 #endif 6176 6177 static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value) 6178 { 6179 if (value == PAGE_COUNTER_MAX) 6180 seq_puts(m, "max\n"); 6181 else 6182 seq_printf(m, "%llu\n", (u64)value * PAGE_SIZE); 6183 6184 return 0; 6185 } 6186 6187 static u64 memory_current_read(struct cgroup_subsys_state *css, 6188 struct cftype *cft) 6189 { 6190 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 6191 6192 return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE; 6193 } 6194 6195 static int memory_min_show(struct seq_file *m, void *v) 6196 { 6197 return seq_puts_memcg_tunable(m, 6198 READ_ONCE(mem_cgroup_from_seq(m)->memory.min)); 6199 } 6200 6201 static ssize_t memory_min_write(struct kernfs_open_file *of, 6202 char *buf, size_t nbytes, loff_t off) 6203 { 6204 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 6205 unsigned long min; 6206 int err; 6207 6208 buf = strstrip(buf); 6209 err = page_counter_memparse(buf, "max", &min); 6210 if (err) 6211 return err; 6212 6213 page_counter_set_min(&memcg->memory, min); 6214 6215 return nbytes; 6216 } 6217 6218 static int memory_low_show(struct seq_file *m, void *v) 6219 { 6220 return seq_puts_memcg_tunable(m, 6221 READ_ONCE(mem_cgroup_from_seq(m)->memory.low)); 6222 } 6223 6224 static ssize_t memory_low_write(struct kernfs_open_file *of, 6225 char *buf, size_t nbytes, loff_t off) 6226 { 6227 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 6228 unsigned long low; 6229 int err; 6230 6231 buf = strstrip(buf); 6232 err = page_counter_memparse(buf, "max", &low); 6233 if (err) 6234 return err; 6235 6236 page_counter_set_low(&memcg->memory, low); 6237 6238 return nbytes; 6239 } 6240 6241 static int memory_high_show(struct seq_file *m, void *v) 6242 { 6243 return seq_puts_memcg_tunable(m, 6244 READ_ONCE(mem_cgroup_from_seq(m)->memory.high)); 6245 } 6246 6247 static ssize_t memory_high_write(struct kernfs_open_file *of, 6248 char *buf, size_t nbytes, loff_t off) 6249 { 6250 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 6251 unsigned int nr_retries = MAX_RECLAIM_RETRIES; 6252 bool drained = false; 6253 unsigned long high; 6254 int err; 6255 6256 buf = strstrip(buf); 6257 err = page_counter_memparse(buf, "max", &high); 6258 if (err) 6259 return err; 6260 6261 page_counter_set_high(&memcg->memory, high); 6262 6263 for (;;) { 6264 unsigned long nr_pages = page_counter_read(&memcg->memory); 6265 unsigned long reclaimed; 6266 6267 if (nr_pages <= high) 6268 break; 6269 6270 if (signal_pending(current)) 6271 break; 6272 6273 if (!drained) { 6274 drain_all_stock(memcg); 6275 drained = true; 6276 continue; 6277 } 6278 6279 reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high, 6280 GFP_KERNEL, true); 6281 6282 if (!reclaimed && !nr_retries--) 6283 break; 6284 } 6285 6286 memcg_wb_domain_size_changed(memcg); 6287 return nbytes; 6288 } 6289 6290 static int memory_max_show(struct seq_file *m, void *v) 6291 { 6292 return seq_puts_memcg_tunable(m, 6293 READ_ONCE(mem_cgroup_from_seq(m)->memory.max)); 6294 } 6295 6296 static ssize_t memory_max_write(struct kernfs_open_file *of, 6297 char *buf, size_t nbytes, loff_t off) 6298 { 6299 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 6300 unsigned int nr_reclaims = MAX_RECLAIM_RETRIES; 6301 bool drained = false; 6302 unsigned long max; 6303 int err; 6304 6305 buf = strstrip(buf); 6306 err = page_counter_memparse(buf, "max", &max); 6307 if (err) 6308 return err; 6309 6310 xchg(&memcg->memory.max, max); 6311 6312 for (;;) { 6313 unsigned long nr_pages = page_counter_read(&memcg->memory); 6314 6315 if (nr_pages <= max) 6316 break; 6317 6318 if (signal_pending(current)) 6319 break; 6320 6321 if (!drained) { 6322 drain_all_stock(memcg); 6323 drained = true; 6324 continue; 6325 } 6326 6327 if (nr_reclaims) { 6328 if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max, 6329 GFP_KERNEL, true)) 6330 nr_reclaims--; 6331 continue; 6332 } 6333 6334 memcg_memory_event(memcg, MEMCG_OOM); 6335 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0)) 6336 break; 6337 } 6338 6339 memcg_wb_domain_size_changed(memcg); 6340 return nbytes; 6341 } 6342 6343 static void __memory_events_show(struct seq_file *m, atomic_long_t *events) 6344 { 6345 seq_printf(m, "low %lu\n", atomic_long_read(&events[MEMCG_LOW])); 6346 seq_printf(m, "high %lu\n", atomic_long_read(&events[MEMCG_HIGH])); 6347 seq_printf(m, "max %lu\n", atomic_long_read(&events[MEMCG_MAX])); 6348 seq_printf(m, "oom %lu\n", atomic_long_read(&events[MEMCG_OOM])); 6349 seq_printf(m, "oom_kill %lu\n", 6350 atomic_long_read(&events[MEMCG_OOM_KILL])); 6351 } 6352 6353 static int memory_events_show(struct seq_file *m, void *v) 6354 { 6355 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 6356 6357 __memory_events_show(m, memcg->memory_events); 6358 return 0; 6359 } 6360 6361 static int memory_events_local_show(struct seq_file *m, void *v) 6362 { 6363 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 6364 6365 __memory_events_show(m, memcg->memory_events_local); 6366 return 0; 6367 } 6368 6369 static int memory_stat_show(struct seq_file *m, void *v) 6370 { 6371 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 6372 char *buf; 6373 6374 buf = memory_stat_format(memcg); 6375 if (!buf) 6376 return -ENOMEM; 6377 seq_puts(m, buf); 6378 kfree(buf); 6379 return 0; 6380 } 6381 6382 #ifdef CONFIG_NUMA 6383 static inline unsigned long lruvec_page_state_output(struct lruvec *lruvec, 6384 int item) 6385 { 6386 return lruvec_page_state(lruvec, item) * memcg_page_state_unit(item); 6387 } 6388 6389 static int memory_numa_stat_show(struct seq_file *m, void *v) 6390 { 6391 int i; 6392 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 6393 6394 cgroup_rstat_flush(memcg->css.cgroup); 6395 6396 for (i = 0; i < ARRAY_SIZE(memory_stats); i++) { 6397 int nid; 6398 6399 if (memory_stats[i].idx >= NR_VM_NODE_STAT_ITEMS) 6400 continue; 6401 6402 seq_printf(m, "%s", memory_stats[i].name); 6403 for_each_node_state(nid, N_MEMORY) { 6404 u64 size; 6405 struct lruvec *lruvec; 6406 6407 lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid)); 6408 size = lruvec_page_state_output(lruvec, 6409 memory_stats[i].idx); 6410 seq_printf(m, " N%d=%llu", nid, size); 6411 } 6412 seq_putc(m, '\n'); 6413 } 6414 6415 return 0; 6416 } 6417 #endif 6418 6419 static int memory_oom_group_show(struct seq_file *m, void *v) 6420 { 6421 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 6422 6423 seq_printf(m, "%d\n", memcg->oom_group); 6424 6425 return 0; 6426 } 6427 6428 static ssize_t memory_oom_group_write(struct kernfs_open_file *of, 6429 char *buf, size_t nbytes, loff_t off) 6430 { 6431 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 6432 int ret, oom_group; 6433 6434 buf = strstrip(buf); 6435 if (!buf) 6436 return -EINVAL; 6437 6438 ret = kstrtoint(buf, 0, &oom_group); 6439 if (ret) 6440 return ret; 6441 6442 if (oom_group != 0 && oom_group != 1) 6443 return -EINVAL; 6444 6445 memcg->oom_group = oom_group; 6446 6447 return nbytes; 6448 } 6449 6450 static struct cftype memory_files[] = { 6451 { 6452 .name = "current", 6453 .flags = CFTYPE_NOT_ON_ROOT, 6454 .read_u64 = memory_current_read, 6455 }, 6456 { 6457 .name = "min", 6458 .flags = CFTYPE_NOT_ON_ROOT, 6459 .seq_show = memory_min_show, 6460 .write = memory_min_write, 6461 }, 6462 { 6463 .name = "low", 6464 .flags = CFTYPE_NOT_ON_ROOT, 6465 .seq_show = memory_low_show, 6466 .write = memory_low_write, 6467 }, 6468 { 6469 .name = "high", 6470 .flags = CFTYPE_NOT_ON_ROOT, 6471 .seq_show = memory_high_show, 6472 .write = memory_high_write, 6473 }, 6474 { 6475 .name = "max", 6476 .flags = CFTYPE_NOT_ON_ROOT, 6477 .seq_show = memory_max_show, 6478 .write = memory_max_write, 6479 }, 6480 { 6481 .name = "events", 6482 .flags = CFTYPE_NOT_ON_ROOT, 6483 .file_offset = offsetof(struct mem_cgroup, events_file), 6484 .seq_show = memory_events_show, 6485 }, 6486 { 6487 .name = "events.local", 6488 .flags = CFTYPE_NOT_ON_ROOT, 6489 .file_offset = offsetof(struct mem_cgroup, events_local_file), 6490 .seq_show = memory_events_local_show, 6491 }, 6492 { 6493 .name = "stat", 6494 .seq_show = memory_stat_show, 6495 }, 6496 #ifdef CONFIG_NUMA 6497 { 6498 .name = "numa_stat", 6499 .seq_show = memory_numa_stat_show, 6500 }, 6501 #endif 6502 { 6503 .name = "oom.group", 6504 .flags = CFTYPE_NOT_ON_ROOT | CFTYPE_NS_DELEGATABLE, 6505 .seq_show = memory_oom_group_show, 6506 .write = memory_oom_group_write, 6507 }, 6508 { } /* terminate */ 6509 }; 6510 6511 struct cgroup_subsys memory_cgrp_subsys = { 6512 .css_alloc = mem_cgroup_css_alloc, 6513 .css_online = mem_cgroup_css_online, 6514 .css_offline = mem_cgroup_css_offline, 6515 .css_released = mem_cgroup_css_released, 6516 .css_free = mem_cgroup_css_free, 6517 .css_reset = mem_cgroup_css_reset, 6518 .css_rstat_flush = mem_cgroup_css_rstat_flush, 6519 .can_attach = mem_cgroup_can_attach, 6520 .cancel_attach = mem_cgroup_cancel_attach, 6521 .post_attach = mem_cgroup_move_task, 6522 .dfl_cftypes = memory_files, 6523 .legacy_cftypes = mem_cgroup_legacy_files, 6524 .early_init = 0, 6525 }; 6526 6527 /* 6528 * This function calculates an individual cgroup's effective 6529 * protection which is derived from its own memory.min/low, its 6530 * parent's and siblings' settings, as well as the actual memory 6531 * distribution in the tree. 6532 * 6533 * The following rules apply to the effective protection values: 6534 * 6535 * 1. At the first level of reclaim, effective protection is equal to 6536 * the declared protection in memory.min and memory.low. 6537 * 6538 * 2. To enable safe delegation of the protection configuration, at 6539 * subsequent levels the effective protection is capped to the 6540 * parent's effective protection. 6541 * 6542 * 3. To make complex and dynamic subtrees easier to configure, the 6543 * user is allowed to overcommit the declared protection at a given 6544 * level. If that is the case, the parent's effective protection is 6545 * distributed to the children in proportion to how much protection 6546 * they have declared and how much of it they are utilizing. 6547 * 6548 * This makes distribution proportional, but also work-conserving: 6549 * if one cgroup claims much more protection than it uses memory, 6550 * the unused remainder is available to its siblings. 6551 * 6552 * 4. Conversely, when the declared protection is undercommitted at a 6553 * given level, the distribution of the larger parental protection 6554 * budget is NOT proportional. A cgroup's protection from a sibling 6555 * is capped to its own memory.min/low setting. 6556 * 6557 * 5. However, to allow protecting recursive subtrees from each other 6558 * without having to declare each individual cgroup's fixed share 6559 * of the ancestor's claim to protection, any unutilized - 6560 * "floating" - protection from up the tree is distributed in 6561 * proportion to each cgroup's *usage*. This makes the protection 6562 * neutral wrt sibling cgroups and lets them compete freely over 6563 * the shared parental protection budget, but it protects the 6564 * subtree as a whole from neighboring subtrees. 6565 * 6566 * Note that 4. and 5. are not in conflict: 4. is about protecting 6567 * against immediate siblings whereas 5. is about protecting against 6568 * neighboring subtrees. 6569 */ 6570 static unsigned long effective_protection(unsigned long usage, 6571 unsigned long parent_usage, 6572 unsigned long setting, 6573 unsigned long parent_effective, 6574 unsigned long siblings_protected) 6575 { 6576 unsigned long protected; 6577 unsigned long ep; 6578 6579 protected = min(usage, setting); 6580 /* 6581 * If all cgroups at this level combined claim and use more 6582 * protection then what the parent affords them, distribute 6583 * shares in proportion to utilization. 6584 * 6585 * We are using actual utilization rather than the statically 6586 * claimed protection in order to be work-conserving: claimed 6587 * but unused protection is available to siblings that would 6588 * otherwise get a smaller chunk than what they claimed. 6589 */ 6590 if (siblings_protected > parent_effective) 6591 return protected * parent_effective / siblings_protected; 6592 6593 /* 6594 * Ok, utilized protection of all children is within what the 6595 * parent affords them, so we know whatever this child claims 6596 * and utilizes is effectively protected. 6597 * 6598 * If there is unprotected usage beyond this value, reclaim 6599 * will apply pressure in proportion to that amount. 6600 * 6601 * If there is unutilized protection, the cgroup will be fully 6602 * shielded from reclaim, but we do return a smaller value for 6603 * protection than what the group could enjoy in theory. This 6604 * is okay. With the overcommit distribution above, effective 6605 * protection is always dependent on how memory is actually 6606 * consumed among the siblings anyway. 6607 */ 6608 ep = protected; 6609 6610 /* 6611 * If the children aren't claiming (all of) the protection 6612 * afforded to them by the parent, distribute the remainder in 6613 * proportion to the (unprotected) memory of each cgroup. That 6614 * way, cgroups that aren't explicitly prioritized wrt each 6615 * other compete freely over the allowance, but they are 6616 * collectively protected from neighboring trees. 6617 * 6618 * We're using unprotected memory for the weight so that if 6619 * some cgroups DO claim explicit protection, we don't protect 6620 * the same bytes twice. 6621 * 6622 * Check both usage and parent_usage against the respective 6623 * protected values. One should imply the other, but they 6624 * aren't read atomically - make sure the division is sane. 6625 */ 6626 if (!(cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT)) 6627 return ep; 6628 if (parent_effective > siblings_protected && 6629 parent_usage > siblings_protected && 6630 usage > protected) { 6631 unsigned long unclaimed; 6632 6633 unclaimed = parent_effective - siblings_protected; 6634 unclaimed *= usage - protected; 6635 unclaimed /= parent_usage - siblings_protected; 6636 6637 ep += unclaimed; 6638 } 6639 6640 return ep; 6641 } 6642 6643 /** 6644 * mem_cgroup_calculate_protection - check if memory consumption is in the normal range 6645 * @root: the top ancestor of the sub-tree being checked 6646 * @memcg: the memory cgroup to check 6647 * 6648 * WARNING: This function is not stateless! It can only be used as part 6649 * of a top-down tree iteration, not for isolated queries. 6650 */ 6651 void mem_cgroup_calculate_protection(struct mem_cgroup *root, 6652 struct mem_cgroup *memcg) 6653 { 6654 unsigned long usage, parent_usage; 6655 struct mem_cgroup *parent; 6656 6657 if (mem_cgroup_disabled()) 6658 return; 6659 6660 if (!root) 6661 root = root_mem_cgroup; 6662 6663 /* 6664 * Effective values of the reclaim targets are ignored so they 6665 * can be stale. Have a look at mem_cgroup_protection for more 6666 * details. 6667 * TODO: calculation should be more robust so that we do not need 6668 * that special casing. 6669 */ 6670 if (memcg == root) 6671 return; 6672 6673 usage = page_counter_read(&memcg->memory); 6674 if (!usage) 6675 return; 6676 6677 parent = parent_mem_cgroup(memcg); 6678 /* No parent means a non-hierarchical mode on v1 memcg */ 6679 if (!parent) 6680 return; 6681 6682 if (parent == root) { 6683 memcg->memory.emin = READ_ONCE(memcg->memory.min); 6684 memcg->memory.elow = READ_ONCE(memcg->memory.low); 6685 return; 6686 } 6687 6688 parent_usage = page_counter_read(&parent->memory); 6689 6690 WRITE_ONCE(memcg->memory.emin, effective_protection(usage, parent_usage, 6691 READ_ONCE(memcg->memory.min), 6692 READ_ONCE(parent->memory.emin), 6693 atomic_long_read(&parent->memory.children_min_usage))); 6694 6695 WRITE_ONCE(memcg->memory.elow, effective_protection(usage, parent_usage, 6696 READ_ONCE(memcg->memory.low), 6697 READ_ONCE(parent->memory.elow), 6698 atomic_long_read(&parent->memory.children_low_usage))); 6699 } 6700 6701 static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg, 6702 gfp_t gfp) 6703 { 6704 long nr_pages = folio_nr_pages(folio); 6705 int ret; 6706 6707 ret = try_charge(memcg, gfp, nr_pages); 6708 if (ret) 6709 goto out; 6710 6711 css_get(&memcg->css); 6712 commit_charge(folio, memcg); 6713 6714 local_irq_disable(); 6715 mem_cgroup_charge_statistics(memcg, nr_pages); 6716 memcg_check_events(memcg, folio_nid(folio)); 6717 local_irq_enable(); 6718 out: 6719 return ret; 6720 } 6721 6722 int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp) 6723 { 6724 struct mem_cgroup *memcg; 6725 int ret; 6726 6727 memcg = get_mem_cgroup_from_mm(mm); 6728 ret = charge_memcg(folio, memcg, gfp); 6729 css_put(&memcg->css); 6730 6731 return ret; 6732 } 6733 6734 /** 6735 * mem_cgroup_swapin_charge_page - charge a newly allocated page for swapin 6736 * @page: page to charge 6737 * @mm: mm context of the victim 6738 * @gfp: reclaim mode 6739 * @entry: swap entry for which the page is allocated 6740 * 6741 * This function charges a page allocated for swapin. Please call this before 6742 * adding the page to the swapcache. 6743 * 6744 * Returns 0 on success. Otherwise, an error code is returned. 6745 */ 6746 int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm, 6747 gfp_t gfp, swp_entry_t entry) 6748 { 6749 struct folio *folio = page_folio(page); 6750 struct mem_cgroup *memcg; 6751 unsigned short id; 6752 int ret; 6753 6754 if (mem_cgroup_disabled()) 6755 return 0; 6756 6757 id = lookup_swap_cgroup_id(entry); 6758 rcu_read_lock(); 6759 memcg = mem_cgroup_from_id(id); 6760 if (!memcg || !css_tryget_online(&memcg->css)) 6761 memcg = get_mem_cgroup_from_mm(mm); 6762 rcu_read_unlock(); 6763 6764 ret = charge_memcg(folio, memcg, gfp); 6765 6766 css_put(&memcg->css); 6767 return ret; 6768 } 6769 6770 /* 6771 * mem_cgroup_swapin_uncharge_swap - uncharge swap slot 6772 * @entry: swap entry for which the page is charged 6773 * 6774 * Call this function after successfully adding the charged page to swapcache. 6775 * 6776 * Note: This function assumes the page for which swap slot is being uncharged 6777 * is order 0 page. 6778 */ 6779 void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry) 6780 { 6781 /* 6782 * Cgroup1's unified memory+swap counter has been charged with the 6783 * new swapcache page, finish the transfer by uncharging the swap 6784 * slot. The swap slot would also get uncharged when it dies, but 6785 * it can stick around indefinitely and we'd count the page twice 6786 * the entire time. 6787 * 6788 * Cgroup2 has separate resource counters for memory and swap, 6789 * so this is a non-issue here. Memory and swap charge lifetimes 6790 * correspond 1:1 to page and swap slot lifetimes: we charge the 6791 * page to memory here, and uncharge swap when the slot is freed. 6792 */ 6793 if (!mem_cgroup_disabled() && do_memsw_account()) { 6794 /* 6795 * The swap entry might not get freed for a long time, 6796 * let's not wait for it. The page already received a 6797 * memory+swap charge, drop the swap entry duplicate. 6798 */ 6799 mem_cgroup_uncharge_swap(entry, 1); 6800 } 6801 } 6802 6803 struct uncharge_gather { 6804 struct mem_cgroup *memcg; 6805 unsigned long nr_memory; 6806 unsigned long pgpgout; 6807 unsigned long nr_kmem; 6808 int nid; 6809 }; 6810 6811 static inline void uncharge_gather_clear(struct uncharge_gather *ug) 6812 { 6813 memset(ug, 0, sizeof(*ug)); 6814 } 6815 6816 static void uncharge_batch(const struct uncharge_gather *ug) 6817 { 6818 unsigned long flags; 6819 6820 if (ug->nr_memory) { 6821 page_counter_uncharge(&ug->memcg->memory, ug->nr_memory); 6822 if (do_memsw_account()) 6823 page_counter_uncharge(&ug->memcg->memsw, ug->nr_memory); 6824 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && ug->nr_kmem) 6825 page_counter_uncharge(&ug->memcg->kmem, ug->nr_kmem); 6826 memcg_oom_recover(ug->memcg); 6827 } 6828 6829 local_irq_save(flags); 6830 __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout); 6831 __this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_memory); 6832 memcg_check_events(ug->memcg, ug->nid); 6833 local_irq_restore(flags); 6834 6835 /* drop reference from uncharge_folio */ 6836 css_put(&ug->memcg->css); 6837 } 6838 6839 static void uncharge_folio(struct folio *folio, struct uncharge_gather *ug) 6840 { 6841 long nr_pages; 6842 struct mem_cgroup *memcg; 6843 struct obj_cgroup *objcg; 6844 bool use_objcg = folio_memcg_kmem(folio); 6845 6846 VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); 6847 6848 /* 6849 * Nobody should be changing or seriously looking at 6850 * folio memcg or objcg at this point, we have fully 6851 * exclusive access to the folio. 6852 */ 6853 if (use_objcg) { 6854 objcg = __folio_objcg(folio); 6855 /* 6856 * This get matches the put at the end of the function and 6857 * kmem pages do not hold memcg references anymore. 6858 */ 6859 memcg = get_mem_cgroup_from_objcg(objcg); 6860 } else { 6861 memcg = __folio_memcg(folio); 6862 } 6863 6864 if (!memcg) 6865 return; 6866 6867 if (ug->memcg != memcg) { 6868 if (ug->memcg) { 6869 uncharge_batch(ug); 6870 uncharge_gather_clear(ug); 6871 } 6872 ug->memcg = memcg; 6873 ug->nid = folio_nid(folio); 6874 6875 /* pairs with css_put in uncharge_batch */ 6876 css_get(&memcg->css); 6877 } 6878 6879 nr_pages = folio_nr_pages(folio); 6880 6881 if (use_objcg) { 6882 ug->nr_memory += nr_pages; 6883 ug->nr_kmem += nr_pages; 6884 6885 folio->memcg_data = 0; 6886 obj_cgroup_put(objcg); 6887 } else { 6888 /* LRU pages aren't accounted at the root level */ 6889 if (!mem_cgroup_is_root(memcg)) 6890 ug->nr_memory += nr_pages; 6891 ug->pgpgout++; 6892 6893 folio->memcg_data = 0; 6894 } 6895 6896 css_put(&memcg->css); 6897 } 6898 6899 void __mem_cgroup_uncharge(struct folio *folio) 6900 { 6901 struct uncharge_gather ug; 6902 6903 /* Don't touch folio->lru of any random page, pre-check: */ 6904 if (!folio_memcg(folio)) 6905 return; 6906 6907 uncharge_gather_clear(&ug); 6908 uncharge_folio(folio, &ug); 6909 uncharge_batch(&ug); 6910 } 6911 6912 /** 6913 * __mem_cgroup_uncharge_list - uncharge a list of page 6914 * @page_list: list of pages to uncharge 6915 * 6916 * Uncharge a list of pages previously charged with 6917 * __mem_cgroup_charge(). 6918 */ 6919 void __mem_cgroup_uncharge_list(struct list_head *page_list) 6920 { 6921 struct uncharge_gather ug; 6922 struct folio *folio; 6923 6924 uncharge_gather_clear(&ug); 6925 list_for_each_entry(folio, page_list, lru) 6926 uncharge_folio(folio, &ug); 6927 if (ug.memcg) 6928 uncharge_batch(&ug); 6929 } 6930 6931 /** 6932 * mem_cgroup_migrate - Charge a folio's replacement. 6933 * @old: Currently circulating folio. 6934 * @new: Replacement folio. 6935 * 6936 * Charge @new as a replacement folio for @old. @old will 6937 * be uncharged upon free. 6938 * 6939 * Both folios must be locked, @new->mapping must be set up. 6940 */ 6941 void mem_cgroup_migrate(struct folio *old, struct folio *new) 6942 { 6943 struct mem_cgroup *memcg; 6944 long nr_pages = folio_nr_pages(new); 6945 unsigned long flags; 6946 6947 VM_BUG_ON_FOLIO(!folio_test_locked(old), old); 6948 VM_BUG_ON_FOLIO(!folio_test_locked(new), new); 6949 VM_BUG_ON_FOLIO(folio_test_anon(old) != folio_test_anon(new), new); 6950 VM_BUG_ON_FOLIO(folio_nr_pages(old) != nr_pages, new); 6951 6952 if (mem_cgroup_disabled()) 6953 return; 6954 6955 /* Page cache replacement: new folio already charged? */ 6956 if (folio_memcg(new)) 6957 return; 6958 6959 memcg = folio_memcg(old); 6960 VM_WARN_ON_ONCE_FOLIO(!memcg, old); 6961 if (!memcg) 6962 return; 6963 6964 /* Force-charge the new page. The old one will be freed soon */ 6965 if (!mem_cgroup_is_root(memcg)) { 6966 page_counter_charge(&memcg->memory, nr_pages); 6967 if (do_memsw_account()) 6968 page_counter_charge(&memcg->memsw, nr_pages); 6969 } 6970 6971 css_get(&memcg->css); 6972 commit_charge(new, memcg); 6973 6974 local_irq_save(flags); 6975 mem_cgroup_charge_statistics(memcg, nr_pages); 6976 memcg_check_events(memcg, folio_nid(new)); 6977 local_irq_restore(flags); 6978 } 6979 6980 DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key); 6981 EXPORT_SYMBOL(memcg_sockets_enabled_key); 6982 6983 void mem_cgroup_sk_alloc(struct sock *sk) 6984 { 6985 struct mem_cgroup *memcg; 6986 6987 if (!mem_cgroup_sockets_enabled) 6988 return; 6989 6990 /* Do not associate the sock with unrelated interrupted task's memcg. */ 6991 if (in_interrupt()) 6992 return; 6993 6994 rcu_read_lock(); 6995 memcg = mem_cgroup_from_task(current); 6996 if (memcg == root_mem_cgroup) 6997 goto out; 6998 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active) 6999 goto out; 7000 if (css_tryget(&memcg->css)) 7001 sk->sk_memcg = memcg; 7002 out: 7003 rcu_read_unlock(); 7004 } 7005 7006 void mem_cgroup_sk_free(struct sock *sk) 7007 { 7008 if (sk->sk_memcg) 7009 css_put(&sk->sk_memcg->css); 7010 } 7011 7012 /** 7013 * mem_cgroup_charge_skmem - charge socket memory 7014 * @memcg: memcg to charge 7015 * @nr_pages: number of pages to charge 7016 * @gfp_mask: reclaim mode 7017 * 7018 * Charges @nr_pages to @memcg. Returns %true if the charge fit within 7019 * @memcg's configured limit, %false if it doesn't. 7020 */ 7021 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages, 7022 gfp_t gfp_mask) 7023 { 7024 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) { 7025 struct page_counter *fail; 7026 7027 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) { 7028 memcg->tcpmem_pressure = 0; 7029 return true; 7030 } 7031 memcg->tcpmem_pressure = 1; 7032 if (gfp_mask & __GFP_NOFAIL) { 7033 page_counter_charge(&memcg->tcpmem, nr_pages); 7034 return true; 7035 } 7036 return false; 7037 } 7038 7039 if (try_charge(memcg, gfp_mask, nr_pages) == 0) { 7040 mod_memcg_state(memcg, MEMCG_SOCK, nr_pages); 7041 return true; 7042 } 7043 7044 return false; 7045 } 7046 7047 /** 7048 * mem_cgroup_uncharge_skmem - uncharge socket memory 7049 * @memcg: memcg to uncharge 7050 * @nr_pages: number of pages to uncharge 7051 */ 7052 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages) 7053 { 7054 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) { 7055 page_counter_uncharge(&memcg->tcpmem, nr_pages); 7056 return; 7057 } 7058 7059 mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages); 7060 7061 refill_stock(memcg, nr_pages); 7062 } 7063 7064 static int __init cgroup_memory(char *s) 7065 { 7066 char *token; 7067 7068 while ((token = strsep(&s, ",")) != NULL) { 7069 if (!*token) 7070 continue; 7071 if (!strcmp(token, "nosocket")) 7072 cgroup_memory_nosocket = true; 7073 if (!strcmp(token, "nokmem")) 7074 cgroup_memory_nokmem = true; 7075 } 7076 return 0; 7077 } 7078 __setup("cgroup.memory=", cgroup_memory); 7079 7080 /* 7081 * subsys_initcall() for memory controller. 7082 * 7083 * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this 7084 * context because of lock dependencies (cgroup_lock -> cpu hotplug) but 7085 * basically everything that doesn't depend on a specific mem_cgroup structure 7086 * should be initialized from here. 7087 */ 7088 static int __init mem_cgroup_init(void) 7089 { 7090 int cpu, node; 7091 7092 /* 7093 * Currently s32 type (can refer to struct batched_lruvec_stat) is 7094 * used for per-memcg-per-cpu caching of per-node statistics. In order 7095 * to work fine, we should make sure that the overfill threshold can't 7096 * exceed S32_MAX / PAGE_SIZE. 7097 */ 7098 BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S32_MAX / PAGE_SIZE); 7099 7100 cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL, 7101 memcg_hotplug_cpu_dead); 7102 7103 for_each_possible_cpu(cpu) 7104 INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work, 7105 drain_local_stock); 7106 7107 for_each_node(node) { 7108 struct mem_cgroup_tree_per_node *rtpn; 7109 7110 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, 7111 node_online(node) ? node : NUMA_NO_NODE); 7112 7113 rtpn->rb_root = RB_ROOT; 7114 rtpn->rb_rightmost = NULL; 7115 spin_lock_init(&rtpn->lock); 7116 soft_limit_tree.rb_tree_per_node[node] = rtpn; 7117 } 7118 7119 return 0; 7120 } 7121 subsys_initcall(mem_cgroup_init); 7122 7123 #ifdef CONFIG_MEMCG_SWAP 7124 static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg) 7125 { 7126 while (!refcount_inc_not_zero(&memcg->id.ref)) { 7127 /* 7128 * The root cgroup cannot be destroyed, so it's refcount must 7129 * always be >= 1. 7130 */ 7131 if (WARN_ON_ONCE(memcg == root_mem_cgroup)) { 7132 VM_BUG_ON(1); 7133 break; 7134 } 7135 memcg = parent_mem_cgroup(memcg); 7136 if (!memcg) 7137 memcg = root_mem_cgroup; 7138 } 7139 return memcg; 7140 } 7141 7142 /** 7143 * mem_cgroup_swapout - transfer a memsw charge to swap 7144 * @page: page whose memsw charge to transfer 7145 * @entry: swap entry to move the charge to 7146 * 7147 * Transfer the memsw charge of @page to @entry. 7148 */ 7149 void mem_cgroup_swapout(struct page *page, swp_entry_t entry) 7150 { 7151 struct mem_cgroup *memcg, *swap_memcg; 7152 unsigned int nr_entries; 7153 unsigned short oldid; 7154 7155 VM_BUG_ON_PAGE(PageLRU(page), page); 7156 VM_BUG_ON_PAGE(page_count(page), page); 7157 7158 if (mem_cgroup_disabled()) 7159 return; 7160 7161 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 7162 return; 7163 7164 memcg = page_memcg(page); 7165 7166 VM_WARN_ON_ONCE_PAGE(!memcg, page); 7167 if (!memcg) 7168 return; 7169 7170 /* 7171 * In case the memcg owning these pages has been offlined and doesn't 7172 * have an ID allocated to it anymore, charge the closest online 7173 * ancestor for the swap instead and transfer the memory+swap charge. 7174 */ 7175 swap_memcg = mem_cgroup_id_get_online(memcg); 7176 nr_entries = thp_nr_pages(page); 7177 /* Get references for the tail pages, too */ 7178 if (nr_entries > 1) 7179 mem_cgroup_id_get_many(swap_memcg, nr_entries - 1); 7180 oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg), 7181 nr_entries); 7182 VM_BUG_ON_PAGE(oldid, page); 7183 mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries); 7184 7185 page->memcg_data = 0; 7186 7187 if (!mem_cgroup_is_root(memcg)) 7188 page_counter_uncharge(&memcg->memory, nr_entries); 7189 7190 if (!cgroup_memory_noswap && memcg != swap_memcg) { 7191 if (!mem_cgroup_is_root(swap_memcg)) 7192 page_counter_charge(&swap_memcg->memsw, nr_entries); 7193 page_counter_uncharge(&memcg->memsw, nr_entries); 7194 } 7195 7196 /* 7197 * Interrupts should be disabled here because the caller holds the 7198 * i_pages lock which is taken with interrupts-off. It is 7199 * important here to have the interrupts disabled because it is the 7200 * only synchronisation we have for updating the per-CPU variables. 7201 */ 7202 VM_BUG_ON(!irqs_disabled()); 7203 mem_cgroup_charge_statistics(memcg, -nr_entries); 7204 memcg_check_events(memcg, page_to_nid(page)); 7205 7206 css_put(&memcg->css); 7207 } 7208 7209 /** 7210 * __mem_cgroup_try_charge_swap - try charging swap space for a page 7211 * @page: page being added to swap 7212 * @entry: swap entry to charge 7213 * 7214 * Try to charge @page's memcg for the swap space at @entry. 7215 * 7216 * Returns 0 on success, -ENOMEM on failure. 7217 */ 7218 int __mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry) 7219 { 7220 unsigned int nr_pages = thp_nr_pages(page); 7221 struct page_counter *counter; 7222 struct mem_cgroup *memcg; 7223 unsigned short oldid; 7224 7225 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) 7226 return 0; 7227 7228 memcg = page_memcg(page); 7229 7230 VM_WARN_ON_ONCE_PAGE(!memcg, page); 7231 if (!memcg) 7232 return 0; 7233 7234 if (!entry.val) { 7235 memcg_memory_event(memcg, MEMCG_SWAP_FAIL); 7236 return 0; 7237 } 7238 7239 memcg = mem_cgroup_id_get_online(memcg); 7240 7241 if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg) && 7242 !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) { 7243 memcg_memory_event(memcg, MEMCG_SWAP_MAX); 7244 memcg_memory_event(memcg, MEMCG_SWAP_FAIL); 7245 mem_cgroup_id_put(memcg); 7246 return -ENOMEM; 7247 } 7248 7249 /* Get references for the tail pages, too */ 7250 if (nr_pages > 1) 7251 mem_cgroup_id_get_many(memcg, nr_pages - 1); 7252 oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages); 7253 VM_BUG_ON_PAGE(oldid, page); 7254 mod_memcg_state(memcg, MEMCG_SWAP, nr_pages); 7255 7256 return 0; 7257 } 7258 7259 /** 7260 * __mem_cgroup_uncharge_swap - uncharge swap space 7261 * @entry: swap entry to uncharge 7262 * @nr_pages: the amount of swap space to uncharge 7263 */ 7264 void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages) 7265 { 7266 struct mem_cgroup *memcg; 7267 unsigned short id; 7268 7269 id = swap_cgroup_record(entry, 0, nr_pages); 7270 rcu_read_lock(); 7271 memcg = mem_cgroup_from_id(id); 7272 if (memcg) { 7273 if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg)) { 7274 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 7275 page_counter_uncharge(&memcg->swap, nr_pages); 7276 else 7277 page_counter_uncharge(&memcg->memsw, nr_pages); 7278 } 7279 mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages); 7280 mem_cgroup_id_put_many(memcg, nr_pages); 7281 } 7282 rcu_read_unlock(); 7283 } 7284 7285 long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg) 7286 { 7287 long nr_swap_pages = get_nr_swap_pages(); 7288 7289 if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys)) 7290 return nr_swap_pages; 7291 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) 7292 nr_swap_pages = min_t(long, nr_swap_pages, 7293 READ_ONCE(memcg->swap.max) - 7294 page_counter_read(&memcg->swap)); 7295 return nr_swap_pages; 7296 } 7297 7298 bool mem_cgroup_swap_full(struct page *page) 7299 { 7300 struct mem_cgroup *memcg; 7301 7302 VM_BUG_ON_PAGE(!PageLocked(page), page); 7303 7304 if (vm_swap_full()) 7305 return true; 7306 if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys)) 7307 return false; 7308 7309 memcg = page_memcg(page); 7310 if (!memcg) 7311 return false; 7312 7313 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) { 7314 unsigned long usage = page_counter_read(&memcg->swap); 7315 7316 if (usage * 2 >= READ_ONCE(memcg->swap.high) || 7317 usage * 2 >= READ_ONCE(memcg->swap.max)) 7318 return true; 7319 } 7320 7321 return false; 7322 } 7323 7324 static int __init setup_swap_account(char *s) 7325 { 7326 if (!strcmp(s, "1")) 7327 cgroup_memory_noswap = false; 7328 else if (!strcmp(s, "0")) 7329 cgroup_memory_noswap = true; 7330 return 1; 7331 } 7332 __setup("swapaccount=", setup_swap_account); 7333 7334 static u64 swap_current_read(struct cgroup_subsys_state *css, 7335 struct cftype *cft) 7336 { 7337 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 7338 7339 return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE; 7340 } 7341 7342 static int swap_high_show(struct seq_file *m, void *v) 7343 { 7344 return seq_puts_memcg_tunable(m, 7345 READ_ONCE(mem_cgroup_from_seq(m)->swap.high)); 7346 } 7347 7348 static ssize_t swap_high_write(struct kernfs_open_file *of, 7349 char *buf, size_t nbytes, loff_t off) 7350 { 7351 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 7352 unsigned long high; 7353 int err; 7354 7355 buf = strstrip(buf); 7356 err = page_counter_memparse(buf, "max", &high); 7357 if (err) 7358 return err; 7359 7360 page_counter_set_high(&memcg->swap, high); 7361 7362 return nbytes; 7363 } 7364 7365 static int swap_max_show(struct seq_file *m, void *v) 7366 { 7367 return seq_puts_memcg_tunable(m, 7368 READ_ONCE(mem_cgroup_from_seq(m)->swap.max)); 7369 } 7370 7371 static ssize_t swap_max_write(struct kernfs_open_file *of, 7372 char *buf, size_t nbytes, loff_t off) 7373 { 7374 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 7375 unsigned long max; 7376 int err; 7377 7378 buf = strstrip(buf); 7379 err = page_counter_memparse(buf, "max", &max); 7380 if (err) 7381 return err; 7382 7383 xchg(&memcg->swap.max, max); 7384 7385 return nbytes; 7386 } 7387 7388 static int swap_events_show(struct seq_file *m, void *v) 7389 { 7390 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 7391 7392 seq_printf(m, "high %lu\n", 7393 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_HIGH])); 7394 seq_printf(m, "max %lu\n", 7395 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX])); 7396 seq_printf(m, "fail %lu\n", 7397 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL])); 7398 7399 return 0; 7400 } 7401 7402 static struct cftype swap_files[] = { 7403 { 7404 .name = "swap.current", 7405 .flags = CFTYPE_NOT_ON_ROOT, 7406 .read_u64 = swap_current_read, 7407 }, 7408 { 7409 .name = "swap.high", 7410 .flags = CFTYPE_NOT_ON_ROOT, 7411 .seq_show = swap_high_show, 7412 .write = swap_high_write, 7413 }, 7414 { 7415 .name = "swap.max", 7416 .flags = CFTYPE_NOT_ON_ROOT, 7417 .seq_show = swap_max_show, 7418 .write = swap_max_write, 7419 }, 7420 { 7421 .name = "swap.events", 7422 .flags = CFTYPE_NOT_ON_ROOT, 7423 .file_offset = offsetof(struct mem_cgroup, swap_events_file), 7424 .seq_show = swap_events_show, 7425 }, 7426 { } /* terminate */ 7427 }; 7428 7429 static struct cftype memsw_files[] = { 7430 { 7431 .name = "memsw.usage_in_bytes", 7432 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE), 7433 .read_u64 = mem_cgroup_read_u64, 7434 }, 7435 { 7436 .name = "memsw.max_usage_in_bytes", 7437 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE), 7438 .write = mem_cgroup_reset, 7439 .read_u64 = mem_cgroup_read_u64, 7440 }, 7441 { 7442 .name = "memsw.limit_in_bytes", 7443 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT), 7444 .write = mem_cgroup_write, 7445 .read_u64 = mem_cgroup_read_u64, 7446 }, 7447 { 7448 .name = "memsw.failcnt", 7449 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT), 7450 .write = mem_cgroup_reset, 7451 .read_u64 = mem_cgroup_read_u64, 7452 }, 7453 { }, /* terminate */ 7454 }; 7455 7456 /* 7457 * If mem_cgroup_swap_init() is implemented as a subsys_initcall() 7458 * instead of a core_initcall(), this could mean cgroup_memory_noswap still 7459 * remains set to false even when memcg is disabled via "cgroup_disable=memory" 7460 * boot parameter. This may result in premature OOPS inside 7461 * mem_cgroup_get_nr_swap_pages() function in corner cases. 7462 */ 7463 static int __init mem_cgroup_swap_init(void) 7464 { 7465 /* No memory control -> no swap control */ 7466 if (mem_cgroup_disabled()) 7467 cgroup_memory_noswap = true; 7468 7469 if (cgroup_memory_noswap) 7470 return 0; 7471 7472 WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, swap_files)); 7473 WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys, memsw_files)); 7474 7475 return 0; 7476 } 7477 core_initcall(mem_cgroup_swap_init); 7478 7479 #endif /* CONFIG_MEMCG_SWAP */ 7480