1 /* memcontrol.c - Memory Controller 2 * 3 * Copyright IBM Corporation, 2007 4 * Author Balbir Singh <balbir@linux.vnet.ibm.com> 5 * 6 * Copyright 2007 OpenVZ SWsoft Inc 7 * Author: Pavel Emelianov <xemul@openvz.org> 8 * 9 * Memory thresholds 10 * Copyright (C) 2009 Nokia Corporation 11 * Author: Kirill A. Shutemov 12 * 13 * Kernel Memory Controller 14 * Copyright (C) 2012 Parallels Inc. and Google Inc. 15 * Authors: Glauber Costa and Suleiman Souhlal 16 * 17 * Native page reclaim 18 * Charge lifetime sanitation 19 * Lockless page tracking & accounting 20 * Unified hierarchy configuration model 21 * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner 22 * 23 * This program is free software; you can redistribute it and/or modify 24 * it under the terms of the GNU General Public License as published by 25 * the Free Software Foundation; either version 2 of the License, or 26 * (at your option) any later version. 27 * 28 * This program is distributed in the hope that it will be useful, 29 * but WITHOUT ANY WARRANTY; without even the implied warranty of 30 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 * GNU General Public License for more details. 32 */ 33 34 #include <linux/page_counter.h> 35 #include <linux/memcontrol.h> 36 #include <linux/cgroup.h> 37 #include <linux/mm.h> 38 #include <linux/sched/mm.h> 39 #include <linux/shmem_fs.h> 40 #include <linux/hugetlb.h> 41 #include <linux/pagemap.h> 42 #include <linux/smp.h> 43 #include <linux/page-flags.h> 44 #include <linux/backing-dev.h> 45 #include <linux/bit_spinlock.h> 46 #include <linux/rcupdate.h> 47 #include <linux/limits.h> 48 #include <linux/export.h> 49 #include <linux/mutex.h> 50 #include <linux/rbtree.h> 51 #include <linux/slab.h> 52 #include <linux/swap.h> 53 #include <linux/swapops.h> 54 #include <linux/spinlock.h> 55 #include <linux/eventfd.h> 56 #include <linux/poll.h> 57 #include <linux/sort.h> 58 #include <linux/fs.h> 59 #include <linux/seq_file.h> 60 #include <linux/vmpressure.h> 61 #include <linux/mm_inline.h> 62 #include <linux/swap_cgroup.h> 63 #include <linux/cpu.h> 64 #include <linux/oom.h> 65 #include <linux/lockdep.h> 66 #include <linux/file.h> 67 #include <linux/tracehook.h> 68 #include "internal.h" 69 #include <net/sock.h> 70 #include <net/ip.h> 71 #include "slab.h" 72 73 #include <linux/uaccess.h> 74 75 #include <trace/events/vmscan.h> 76 77 struct cgroup_subsys memory_cgrp_subsys __read_mostly; 78 EXPORT_SYMBOL(memory_cgrp_subsys); 79 80 struct mem_cgroup *root_mem_cgroup __read_mostly; 81 82 #define MEM_CGROUP_RECLAIM_RETRIES 5 83 84 /* Socket memory accounting disabled? */ 85 static bool cgroup_memory_nosocket; 86 87 /* Kernel memory accounting disabled? */ 88 static bool cgroup_memory_nokmem; 89 90 /* Whether the swap controller is active */ 91 #ifdef CONFIG_MEMCG_SWAP 92 int do_swap_account __read_mostly; 93 #else 94 #define do_swap_account 0 95 #endif 96 97 /* Whether legacy memory+swap accounting is active */ 98 static bool do_memsw_account(void) 99 { 100 return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && do_swap_account; 101 } 102 103 static const char *const mem_cgroup_lru_names[] = { 104 "inactive_anon", 105 "active_anon", 106 "inactive_file", 107 "active_file", 108 "unevictable", 109 }; 110 111 #define THRESHOLDS_EVENTS_TARGET 128 112 #define SOFTLIMIT_EVENTS_TARGET 1024 113 #define NUMAINFO_EVENTS_TARGET 1024 114 115 /* 116 * Cgroups above their limits are maintained in a RB-Tree, independent of 117 * their hierarchy representation 118 */ 119 120 struct mem_cgroup_tree_per_node { 121 struct rb_root rb_root; 122 struct rb_node *rb_rightmost; 123 spinlock_t lock; 124 }; 125 126 struct mem_cgroup_tree { 127 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES]; 128 }; 129 130 static struct mem_cgroup_tree soft_limit_tree __read_mostly; 131 132 /* for OOM */ 133 struct mem_cgroup_eventfd_list { 134 struct list_head list; 135 struct eventfd_ctx *eventfd; 136 }; 137 138 /* 139 * cgroup_event represents events which userspace want to receive. 140 */ 141 struct mem_cgroup_event { 142 /* 143 * memcg which the event belongs to. 144 */ 145 struct mem_cgroup *memcg; 146 /* 147 * eventfd to signal userspace about the event. 148 */ 149 struct eventfd_ctx *eventfd; 150 /* 151 * Each of these stored in a list by the cgroup. 152 */ 153 struct list_head list; 154 /* 155 * register_event() callback will be used to add new userspace 156 * waiter for changes related to this event. Use eventfd_signal() 157 * on eventfd to send notification to userspace. 158 */ 159 int (*register_event)(struct mem_cgroup *memcg, 160 struct eventfd_ctx *eventfd, const char *args); 161 /* 162 * unregister_event() callback will be called when userspace closes 163 * the eventfd or on cgroup removing. This callback must be set, 164 * if you want provide notification functionality. 165 */ 166 void (*unregister_event)(struct mem_cgroup *memcg, 167 struct eventfd_ctx *eventfd); 168 /* 169 * All fields below needed to unregister event when 170 * userspace closes eventfd. 171 */ 172 poll_table pt; 173 wait_queue_head_t *wqh; 174 wait_queue_entry_t wait; 175 struct work_struct remove; 176 }; 177 178 static void mem_cgroup_threshold(struct mem_cgroup *memcg); 179 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg); 180 181 /* Stuffs for move charges at task migration. */ 182 /* 183 * Types of charges to be moved. 184 */ 185 #define MOVE_ANON 0x1U 186 #define MOVE_FILE 0x2U 187 #define MOVE_MASK (MOVE_ANON | MOVE_FILE) 188 189 /* "mc" and its members are protected by cgroup_mutex */ 190 static struct move_charge_struct { 191 spinlock_t lock; /* for from, to */ 192 struct mm_struct *mm; 193 struct mem_cgroup *from; 194 struct mem_cgroup *to; 195 unsigned long flags; 196 unsigned long precharge; 197 unsigned long moved_charge; 198 unsigned long moved_swap; 199 struct task_struct *moving_task; /* a task moving charges */ 200 wait_queue_head_t waitq; /* a waitq for other context */ 201 } mc = { 202 .lock = __SPIN_LOCK_UNLOCKED(mc.lock), 203 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq), 204 }; 205 206 /* 207 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft 208 * limit reclaim to prevent infinite loops, if they ever occur. 209 */ 210 #define MEM_CGROUP_MAX_RECLAIM_LOOPS 100 211 #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2 212 213 enum charge_type { 214 MEM_CGROUP_CHARGE_TYPE_CACHE = 0, 215 MEM_CGROUP_CHARGE_TYPE_ANON, 216 MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */ 217 MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */ 218 NR_CHARGE_TYPE, 219 }; 220 221 /* for encoding cft->private value on file */ 222 enum res_type { 223 _MEM, 224 _MEMSWAP, 225 _OOM_TYPE, 226 _KMEM, 227 _TCP, 228 }; 229 230 #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val)) 231 #define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff) 232 #define MEMFILE_ATTR(val) ((val) & 0xffff) 233 /* Used for OOM nofiier */ 234 #define OOM_CONTROL (0) 235 236 /* Some nice accessors for the vmpressure. */ 237 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg) 238 { 239 if (!memcg) 240 memcg = root_mem_cgroup; 241 return &memcg->vmpressure; 242 } 243 244 struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr) 245 { 246 return &container_of(vmpr, struct mem_cgroup, vmpressure)->css; 247 } 248 249 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg) 250 { 251 return (memcg == root_mem_cgroup); 252 } 253 254 #ifndef CONFIG_SLOB 255 /* 256 * This will be the memcg's index in each cache's ->memcg_params.memcg_caches. 257 * The main reason for not using cgroup id for this: 258 * this works better in sparse environments, where we have a lot of memcgs, 259 * but only a few kmem-limited. Or also, if we have, for instance, 200 260 * memcgs, and none but the 200th is kmem-limited, we'd have to have a 261 * 200 entry array for that. 262 * 263 * The current size of the caches array is stored in memcg_nr_cache_ids. It 264 * will double each time we have to increase it. 265 */ 266 static DEFINE_IDA(memcg_cache_ida); 267 int memcg_nr_cache_ids; 268 269 /* Protects memcg_nr_cache_ids */ 270 static DECLARE_RWSEM(memcg_cache_ids_sem); 271 272 void memcg_get_cache_ids(void) 273 { 274 down_read(&memcg_cache_ids_sem); 275 } 276 277 void memcg_put_cache_ids(void) 278 { 279 up_read(&memcg_cache_ids_sem); 280 } 281 282 /* 283 * MIN_SIZE is different than 1, because we would like to avoid going through 284 * the alloc/free process all the time. In a small machine, 4 kmem-limited 285 * cgroups is a reasonable guess. In the future, it could be a parameter or 286 * tunable, but that is strictly not necessary. 287 * 288 * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get 289 * this constant directly from cgroup, but it is understandable that this is 290 * better kept as an internal representation in cgroup.c. In any case, the 291 * cgrp_id space is not getting any smaller, and we don't have to necessarily 292 * increase ours as well if it increases. 293 */ 294 #define MEMCG_CACHES_MIN_SIZE 4 295 #define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX 296 297 /* 298 * A lot of the calls to the cache allocation functions are expected to be 299 * inlined by the compiler. Since the calls to memcg_kmem_get_cache are 300 * conditional to this static branch, we'll have to allow modules that does 301 * kmem_cache_alloc and the such to see this symbol as well 302 */ 303 DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key); 304 EXPORT_SYMBOL(memcg_kmem_enabled_key); 305 306 struct workqueue_struct *memcg_kmem_cache_wq; 307 308 #endif /* !CONFIG_SLOB */ 309 310 /** 311 * mem_cgroup_css_from_page - css of the memcg associated with a page 312 * @page: page of interest 313 * 314 * If memcg is bound to the default hierarchy, css of the memcg associated 315 * with @page is returned. The returned css remains associated with @page 316 * until it is released. 317 * 318 * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup 319 * is returned. 320 */ 321 struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page) 322 { 323 struct mem_cgroup *memcg; 324 325 memcg = page->mem_cgroup; 326 327 if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys)) 328 memcg = root_mem_cgroup; 329 330 return &memcg->css; 331 } 332 333 /** 334 * page_cgroup_ino - return inode number of the memcg a page is charged to 335 * @page: the page 336 * 337 * Look up the closest online ancestor of the memory cgroup @page is charged to 338 * and return its inode number or 0 if @page is not charged to any cgroup. It 339 * is safe to call this function without holding a reference to @page. 340 * 341 * Note, this function is inherently racy, because there is nothing to prevent 342 * the cgroup inode from getting torn down and potentially reallocated a moment 343 * after page_cgroup_ino() returns, so it only should be used by callers that 344 * do not care (such as procfs interfaces). 345 */ 346 ino_t page_cgroup_ino(struct page *page) 347 { 348 struct mem_cgroup *memcg; 349 unsigned long ino = 0; 350 351 rcu_read_lock(); 352 memcg = READ_ONCE(page->mem_cgroup); 353 while (memcg && !(memcg->css.flags & CSS_ONLINE)) 354 memcg = parent_mem_cgroup(memcg); 355 if (memcg) 356 ino = cgroup_ino(memcg->css.cgroup); 357 rcu_read_unlock(); 358 return ino; 359 } 360 361 static struct mem_cgroup_per_node * 362 mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page) 363 { 364 int nid = page_to_nid(page); 365 366 return memcg->nodeinfo[nid]; 367 } 368 369 static struct mem_cgroup_tree_per_node * 370 soft_limit_tree_node(int nid) 371 { 372 return soft_limit_tree.rb_tree_per_node[nid]; 373 } 374 375 static struct mem_cgroup_tree_per_node * 376 soft_limit_tree_from_page(struct page *page) 377 { 378 int nid = page_to_nid(page); 379 380 return soft_limit_tree.rb_tree_per_node[nid]; 381 } 382 383 static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz, 384 struct mem_cgroup_tree_per_node *mctz, 385 unsigned long new_usage_in_excess) 386 { 387 struct rb_node **p = &mctz->rb_root.rb_node; 388 struct rb_node *parent = NULL; 389 struct mem_cgroup_per_node *mz_node; 390 bool rightmost = true; 391 392 if (mz->on_tree) 393 return; 394 395 mz->usage_in_excess = new_usage_in_excess; 396 if (!mz->usage_in_excess) 397 return; 398 while (*p) { 399 parent = *p; 400 mz_node = rb_entry(parent, struct mem_cgroup_per_node, 401 tree_node); 402 if (mz->usage_in_excess < mz_node->usage_in_excess) { 403 p = &(*p)->rb_left; 404 rightmost = false; 405 } 406 407 /* 408 * We can't avoid mem cgroups that are over their soft 409 * limit by the same amount 410 */ 411 else if (mz->usage_in_excess >= mz_node->usage_in_excess) 412 p = &(*p)->rb_right; 413 } 414 415 if (rightmost) 416 mctz->rb_rightmost = &mz->tree_node; 417 418 rb_link_node(&mz->tree_node, parent, p); 419 rb_insert_color(&mz->tree_node, &mctz->rb_root); 420 mz->on_tree = true; 421 } 422 423 static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz, 424 struct mem_cgroup_tree_per_node *mctz) 425 { 426 if (!mz->on_tree) 427 return; 428 429 if (&mz->tree_node == mctz->rb_rightmost) 430 mctz->rb_rightmost = rb_prev(&mz->tree_node); 431 432 rb_erase(&mz->tree_node, &mctz->rb_root); 433 mz->on_tree = false; 434 } 435 436 static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz, 437 struct mem_cgroup_tree_per_node *mctz) 438 { 439 unsigned long flags; 440 441 spin_lock_irqsave(&mctz->lock, flags); 442 __mem_cgroup_remove_exceeded(mz, mctz); 443 spin_unlock_irqrestore(&mctz->lock, flags); 444 } 445 446 static unsigned long soft_limit_excess(struct mem_cgroup *memcg) 447 { 448 unsigned long nr_pages = page_counter_read(&memcg->memory); 449 unsigned long soft_limit = READ_ONCE(memcg->soft_limit); 450 unsigned long excess = 0; 451 452 if (nr_pages > soft_limit) 453 excess = nr_pages - soft_limit; 454 455 return excess; 456 } 457 458 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page) 459 { 460 unsigned long excess; 461 struct mem_cgroup_per_node *mz; 462 struct mem_cgroup_tree_per_node *mctz; 463 464 mctz = soft_limit_tree_from_page(page); 465 if (!mctz) 466 return; 467 /* 468 * Necessary to update all ancestors when hierarchy is used. 469 * because their event counter is not touched. 470 */ 471 for (; memcg; memcg = parent_mem_cgroup(memcg)) { 472 mz = mem_cgroup_page_nodeinfo(memcg, page); 473 excess = soft_limit_excess(memcg); 474 /* 475 * We have to update the tree if mz is on RB-tree or 476 * mem is over its softlimit. 477 */ 478 if (excess || mz->on_tree) { 479 unsigned long flags; 480 481 spin_lock_irqsave(&mctz->lock, flags); 482 /* if on-tree, remove it */ 483 if (mz->on_tree) 484 __mem_cgroup_remove_exceeded(mz, mctz); 485 /* 486 * Insert again. mz->usage_in_excess will be updated. 487 * If excess is 0, no tree ops. 488 */ 489 __mem_cgroup_insert_exceeded(mz, mctz, excess); 490 spin_unlock_irqrestore(&mctz->lock, flags); 491 } 492 } 493 } 494 495 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg) 496 { 497 struct mem_cgroup_tree_per_node *mctz; 498 struct mem_cgroup_per_node *mz; 499 int nid; 500 501 for_each_node(nid) { 502 mz = mem_cgroup_nodeinfo(memcg, nid); 503 mctz = soft_limit_tree_node(nid); 504 if (mctz) 505 mem_cgroup_remove_exceeded(mz, mctz); 506 } 507 } 508 509 static struct mem_cgroup_per_node * 510 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz) 511 { 512 struct mem_cgroup_per_node *mz; 513 514 retry: 515 mz = NULL; 516 if (!mctz->rb_rightmost) 517 goto done; /* Nothing to reclaim from */ 518 519 mz = rb_entry(mctz->rb_rightmost, 520 struct mem_cgroup_per_node, tree_node); 521 /* 522 * Remove the node now but someone else can add it back, 523 * we will to add it back at the end of reclaim to its correct 524 * position in the tree. 525 */ 526 __mem_cgroup_remove_exceeded(mz, mctz); 527 if (!soft_limit_excess(mz->memcg) || 528 !css_tryget_online(&mz->memcg->css)) 529 goto retry; 530 done: 531 return mz; 532 } 533 534 static struct mem_cgroup_per_node * 535 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz) 536 { 537 struct mem_cgroup_per_node *mz; 538 539 spin_lock_irq(&mctz->lock); 540 mz = __mem_cgroup_largest_soft_limit_node(mctz); 541 spin_unlock_irq(&mctz->lock); 542 return mz; 543 } 544 545 /* 546 * Return page count for single (non recursive) @memcg. 547 * 548 * Implementation Note: reading percpu statistics for memcg. 549 * 550 * Both of vmstat[] and percpu_counter has threshold and do periodic 551 * synchronization to implement "quick" read. There are trade-off between 552 * reading cost and precision of value. Then, we may have a chance to implement 553 * a periodic synchronization of counter in memcg's counter. 554 * 555 * But this _read() function is used for user interface now. The user accounts 556 * memory usage by memory cgroup and he _always_ requires exact value because 557 * he accounts memory. Even if we provide quick-and-fuzzy read, we always 558 * have to visit all online cpus and make sum. So, for now, unnecessary 559 * synchronization is not implemented. (just implemented for cpu hotplug) 560 * 561 * If there are kernel internal actions which can make use of some not-exact 562 * value, and reading all cpu value can be performance bottleneck in some 563 * common workload, threshold and synchronization as vmstat[] should be 564 * implemented. 565 * 566 * The parameter idx can be of type enum memcg_event_item or vm_event_item. 567 */ 568 569 static unsigned long memcg_sum_events(struct mem_cgroup *memcg, 570 int event) 571 { 572 unsigned long val = 0; 573 int cpu; 574 575 for_each_possible_cpu(cpu) 576 val += per_cpu(memcg->stat->events[event], cpu); 577 return val; 578 } 579 580 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg, 581 struct page *page, 582 bool compound, int nr_pages) 583 { 584 /* 585 * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is 586 * counted as CACHE even if it's on ANON LRU. 587 */ 588 if (PageAnon(page)) 589 __this_cpu_add(memcg->stat->count[MEMCG_RSS], nr_pages); 590 else { 591 __this_cpu_add(memcg->stat->count[MEMCG_CACHE], nr_pages); 592 if (PageSwapBacked(page)) 593 __this_cpu_add(memcg->stat->count[NR_SHMEM], nr_pages); 594 } 595 596 if (compound) { 597 VM_BUG_ON_PAGE(!PageTransHuge(page), page); 598 __this_cpu_add(memcg->stat->count[MEMCG_RSS_HUGE], nr_pages); 599 } 600 601 /* pagein of a big page is an event. So, ignore page size */ 602 if (nr_pages > 0) 603 __this_cpu_inc(memcg->stat->events[PGPGIN]); 604 else { 605 __this_cpu_inc(memcg->stat->events[PGPGOUT]); 606 nr_pages = -nr_pages; /* for event */ 607 } 608 609 __this_cpu_add(memcg->stat->nr_page_events, nr_pages); 610 } 611 612 unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg, 613 int nid, unsigned int lru_mask) 614 { 615 struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg); 616 unsigned long nr = 0; 617 enum lru_list lru; 618 619 VM_BUG_ON((unsigned)nid >= nr_node_ids); 620 621 for_each_lru(lru) { 622 if (!(BIT(lru) & lru_mask)) 623 continue; 624 nr += mem_cgroup_get_lru_size(lruvec, lru); 625 } 626 return nr; 627 } 628 629 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg, 630 unsigned int lru_mask) 631 { 632 unsigned long nr = 0; 633 int nid; 634 635 for_each_node_state(nid, N_MEMORY) 636 nr += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask); 637 return nr; 638 } 639 640 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg, 641 enum mem_cgroup_events_target target) 642 { 643 unsigned long val, next; 644 645 val = __this_cpu_read(memcg->stat->nr_page_events); 646 next = __this_cpu_read(memcg->stat->targets[target]); 647 /* from time_after() in jiffies.h */ 648 if ((long)(next - val) < 0) { 649 switch (target) { 650 case MEM_CGROUP_TARGET_THRESH: 651 next = val + THRESHOLDS_EVENTS_TARGET; 652 break; 653 case MEM_CGROUP_TARGET_SOFTLIMIT: 654 next = val + SOFTLIMIT_EVENTS_TARGET; 655 break; 656 case MEM_CGROUP_TARGET_NUMAINFO: 657 next = val + NUMAINFO_EVENTS_TARGET; 658 break; 659 default: 660 break; 661 } 662 __this_cpu_write(memcg->stat->targets[target], next); 663 return true; 664 } 665 return false; 666 } 667 668 /* 669 * Check events in order. 670 * 671 */ 672 static void memcg_check_events(struct mem_cgroup *memcg, struct page *page) 673 { 674 /* threshold event is triggered in finer grain than soft limit */ 675 if (unlikely(mem_cgroup_event_ratelimit(memcg, 676 MEM_CGROUP_TARGET_THRESH))) { 677 bool do_softlimit; 678 bool do_numainfo __maybe_unused; 679 680 do_softlimit = mem_cgroup_event_ratelimit(memcg, 681 MEM_CGROUP_TARGET_SOFTLIMIT); 682 #if MAX_NUMNODES > 1 683 do_numainfo = mem_cgroup_event_ratelimit(memcg, 684 MEM_CGROUP_TARGET_NUMAINFO); 685 #endif 686 mem_cgroup_threshold(memcg); 687 if (unlikely(do_softlimit)) 688 mem_cgroup_update_tree(memcg, page); 689 #if MAX_NUMNODES > 1 690 if (unlikely(do_numainfo)) 691 atomic_inc(&memcg->numainfo_events); 692 #endif 693 } 694 } 695 696 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) 697 { 698 /* 699 * mm_update_next_owner() may clear mm->owner to NULL 700 * if it races with swapoff, page migration, etc. 701 * So this can be called with p == NULL. 702 */ 703 if (unlikely(!p)) 704 return NULL; 705 706 return mem_cgroup_from_css(task_css(p, memory_cgrp_id)); 707 } 708 EXPORT_SYMBOL(mem_cgroup_from_task); 709 710 static struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm) 711 { 712 struct mem_cgroup *memcg = NULL; 713 714 rcu_read_lock(); 715 do { 716 /* 717 * Page cache insertions can happen withou an 718 * actual mm context, e.g. during disk probing 719 * on boot, loopback IO, acct() writes etc. 720 */ 721 if (unlikely(!mm)) 722 memcg = root_mem_cgroup; 723 else { 724 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); 725 if (unlikely(!memcg)) 726 memcg = root_mem_cgroup; 727 } 728 } while (!css_tryget_online(&memcg->css)); 729 rcu_read_unlock(); 730 return memcg; 731 } 732 733 /** 734 * mem_cgroup_iter - iterate over memory cgroup hierarchy 735 * @root: hierarchy root 736 * @prev: previously returned memcg, NULL on first invocation 737 * @reclaim: cookie for shared reclaim walks, NULL for full walks 738 * 739 * Returns references to children of the hierarchy below @root, or 740 * @root itself, or %NULL after a full round-trip. 741 * 742 * Caller must pass the return value in @prev on subsequent 743 * invocations for reference counting, or use mem_cgroup_iter_break() 744 * to cancel a hierarchy walk before the round-trip is complete. 745 * 746 * Reclaimers can specify a zone and a priority level in @reclaim to 747 * divide up the memcgs in the hierarchy among all concurrent 748 * reclaimers operating on the same zone and priority. 749 */ 750 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root, 751 struct mem_cgroup *prev, 752 struct mem_cgroup_reclaim_cookie *reclaim) 753 { 754 struct mem_cgroup_reclaim_iter *uninitialized_var(iter); 755 struct cgroup_subsys_state *css = NULL; 756 struct mem_cgroup *memcg = NULL; 757 struct mem_cgroup *pos = NULL; 758 759 if (mem_cgroup_disabled()) 760 return NULL; 761 762 if (!root) 763 root = root_mem_cgroup; 764 765 if (prev && !reclaim) 766 pos = prev; 767 768 if (!root->use_hierarchy && root != root_mem_cgroup) { 769 if (prev) 770 goto out; 771 return root; 772 } 773 774 rcu_read_lock(); 775 776 if (reclaim) { 777 struct mem_cgroup_per_node *mz; 778 779 mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id); 780 iter = &mz->iter[reclaim->priority]; 781 782 if (prev && reclaim->generation != iter->generation) 783 goto out_unlock; 784 785 while (1) { 786 pos = READ_ONCE(iter->position); 787 if (!pos || css_tryget(&pos->css)) 788 break; 789 /* 790 * css reference reached zero, so iter->position will 791 * be cleared by ->css_released. However, we should not 792 * rely on this happening soon, because ->css_released 793 * is called from a work queue, and by busy-waiting we 794 * might block it. So we clear iter->position right 795 * away. 796 */ 797 (void)cmpxchg(&iter->position, pos, NULL); 798 } 799 } 800 801 if (pos) 802 css = &pos->css; 803 804 for (;;) { 805 css = css_next_descendant_pre(css, &root->css); 806 if (!css) { 807 /* 808 * Reclaimers share the hierarchy walk, and a 809 * new one might jump in right at the end of 810 * the hierarchy - make sure they see at least 811 * one group and restart from the beginning. 812 */ 813 if (!prev) 814 continue; 815 break; 816 } 817 818 /* 819 * Verify the css and acquire a reference. The root 820 * is provided by the caller, so we know it's alive 821 * and kicking, and don't take an extra reference. 822 */ 823 memcg = mem_cgroup_from_css(css); 824 825 if (css == &root->css) 826 break; 827 828 if (css_tryget(css)) 829 break; 830 831 memcg = NULL; 832 } 833 834 if (reclaim) { 835 /* 836 * The position could have already been updated by a competing 837 * thread, so check that the value hasn't changed since we read 838 * it to avoid reclaiming from the same cgroup twice. 839 */ 840 (void)cmpxchg(&iter->position, pos, memcg); 841 842 if (pos) 843 css_put(&pos->css); 844 845 if (!memcg) 846 iter->generation++; 847 else if (!prev) 848 reclaim->generation = iter->generation; 849 } 850 851 out_unlock: 852 rcu_read_unlock(); 853 out: 854 if (prev && prev != root) 855 css_put(&prev->css); 856 857 return memcg; 858 } 859 860 /** 861 * mem_cgroup_iter_break - abort a hierarchy walk prematurely 862 * @root: hierarchy root 863 * @prev: last visited hierarchy member as returned by mem_cgroup_iter() 864 */ 865 void mem_cgroup_iter_break(struct mem_cgroup *root, 866 struct mem_cgroup *prev) 867 { 868 if (!root) 869 root = root_mem_cgroup; 870 if (prev && prev != root) 871 css_put(&prev->css); 872 } 873 874 static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg) 875 { 876 struct mem_cgroup *memcg = dead_memcg; 877 struct mem_cgroup_reclaim_iter *iter; 878 struct mem_cgroup_per_node *mz; 879 int nid; 880 int i; 881 882 while ((memcg = parent_mem_cgroup(memcg))) { 883 for_each_node(nid) { 884 mz = mem_cgroup_nodeinfo(memcg, nid); 885 for (i = 0; i <= DEF_PRIORITY; i++) { 886 iter = &mz->iter[i]; 887 cmpxchg(&iter->position, 888 dead_memcg, NULL); 889 } 890 } 891 } 892 } 893 894 /* 895 * Iteration constructs for visiting all cgroups (under a tree). If 896 * loops are exited prematurely (break), mem_cgroup_iter_break() must 897 * be used for reference counting. 898 */ 899 #define for_each_mem_cgroup_tree(iter, root) \ 900 for (iter = mem_cgroup_iter(root, NULL, NULL); \ 901 iter != NULL; \ 902 iter = mem_cgroup_iter(root, iter, NULL)) 903 904 #define for_each_mem_cgroup(iter) \ 905 for (iter = mem_cgroup_iter(NULL, NULL, NULL); \ 906 iter != NULL; \ 907 iter = mem_cgroup_iter(NULL, iter, NULL)) 908 909 /** 910 * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy 911 * @memcg: hierarchy root 912 * @fn: function to call for each task 913 * @arg: argument passed to @fn 914 * 915 * This function iterates over tasks attached to @memcg or to any of its 916 * descendants and calls @fn for each task. If @fn returns a non-zero 917 * value, the function breaks the iteration loop and returns the value. 918 * Otherwise, it will iterate over all tasks and return 0. 919 * 920 * This function must not be called for the root memory cgroup. 921 */ 922 int mem_cgroup_scan_tasks(struct mem_cgroup *memcg, 923 int (*fn)(struct task_struct *, void *), void *arg) 924 { 925 struct mem_cgroup *iter; 926 int ret = 0; 927 928 BUG_ON(memcg == root_mem_cgroup); 929 930 for_each_mem_cgroup_tree(iter, memcg) { 931 struct css_task_iter it; 932 struct task_struct *task; 933 934 css_task_iter_start(&iter->css, 0, &it); 935 while (!ret && (task = css_task_iter_next(&it))) 936 ret = fn(task, arg); 937 css_task_iter_end(&it); 938 if (ret) { 939 mem_cgroup_iter_break(memcg, iter); 940 break; 941 } 942 } 943 return ret; 944 } 945 946 /** 947 * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page 948 * @page: the page 949 * @zone: zone of the page 950 * 951 * This function is only safe when following the LRU page isolation 952 * and putback protocol: the LRU lock must be held, and the page must 953 * either be PageLRU() or the caller must have isolated/allocated it. 954 */ 955 struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgdat) 956 { 957 struct mem_cgroup_per_node *mz; 958 struct mem_cgroup *memcg; 959 struct lruvec *lruvec; 960 961 if (mem_cgroup_disabled()) { 962 lruvec = &pgdat->lruvec; 963 goto out; 964 } 965 966 memcg = page->mem_cgroup; 967 /* 968 * Swapcache readahead pages are added to the LRU - and 969 * possibly migrated - before they are charged. 970 */ 971 if (!memcg) 972 memcg = root_mem_cgroup; 973 974 mz = mem_cgroup_page_nodeinfo(memcg, page); 975 lruvec = &mz->lruvec; 976 out: 977 /* 978 * Since a node can be onlined after the mem_cgroup was created, 979 * we have to be prepared to initialize lruvec->zone here; 980 * and if offlined then reonlined, we need to reinitialize it. 981 */ 982 if (unlikely(lruvec->pgdat != pgdat)) 983 lruvec->pgdat = pgdat; 984 return lruvec; 985 } 986 987 /** 988 * mem_cgroup_update_lru_size - account for adding or removing an lru page 989 * @lruvec: mem_cgroup per zone lru vector 990 * @lru: index of lru list the page is sitting on 991 * @zid: zone id of the accounted pages 992 * @nr_pages: positive when adding or negative when removing 993 * 994 * This function must be called under lru_lock, just before a page is added 995 * to or just after a page is removed from an lru list (that ordering being 996 * so as to allow it to check that lru_size 0 is consistent with list_empty). 997 */ 998 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, 999 int zid, int nr_pages) 1000 { 1001 struct mem_cgroup_per_node *mz; 1002 unsigned long *lru_size; 1003 long size; 1004 1005 if (mem_cgroup_disabled()) 1006 return; 1007 1008 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); 1009 lru_size = &mz->lru_zone_size[zid][lru]; 1010 1011 if (nr_pages < 0) 1012 *lru_size += nr_pages; 1013 1014 size = *lru_size; 1015 if (WARN_ONCE(size < 0, 1016 "%s(%p, %d, %d): lru_size %ld\n", 1017 __func__, lruvec, lru, nr_pages, size)) { 1018 VM_BUG_ON(1); 1019 *lru_size = 0; 1020 } 1021 1022 if (nr_pages > 0) 1023 *lru_size += nr_pages; 1024 } 1025 1026 bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg) 1027 { 1028 struct mem_cgroup *task_memcg; 1029 struct task_struct *p; 1030 bool ret; 1031 1032 p = find_lock_task_mm(task); 1033 if (p) { 1034 task_memcg = get_mem_cgroup_from_mm(p->mm); 1035 task_unlock(p); 1036 } else { 1037 /* 1038 * All threads may have already detached their mm's, but the oom 1039 * killer still needs to detect if they have already been oom 1040 * killed to prevent needlessly killing additional tasks. 1041 */ 1042 rcu_read_lock(); 1043 task_memcg = mem_cgroup_from_task(task); 1044 css_get(&task_memcg->css); 1045 rcu_read_unlock(); 1046 } 1047 ret = mem_cgroup_is_descendant(task_memcg, memcg); 1048 css_put(&task_memcg->css); 1049 return ret; 1050 } 1051 1052 /** 1053 * mem_cgroup_margin - calculate chargeable space of a memory cgroup 1054 * @memcg: the memory cgroup 1055 * 1056 * Returns the maximum amount of memory @mem can be charged with, in 1057 * pages. 1058 */ 1059 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg) 1060 { 1061 unsigned long margin = 0; 1062 unsigned long count; 1063 unsigned long limit; 1064 1065 count = page_counter_read(&memcg->memory); 1066 limit = READ_ONCE(memcg->memory.limit); 1067 if (count < limit) 1068 margin = limit - count; 1069 1070 if (do_memsw_account()) { 1071 count = page_counter_read(&memcg->memsw); 1072 limit = READ_ONCE(memcg->memsw.limit); 1073 if (count <= limit) 1074 margin = min(margin, limit - count); 1075 else 1076 margin = 0; 1077 } 1078 1079 return margin; 1080 } 1081 1082 /* 1083 * A routine for checking "mem" is under move_account() or not. 1084 * 1085 * Checking a cgroup is mc.from or mc.to or under hierarchy of 1086 * moving cgroups. This is for waiting at high-memory pressure 1087 * caused by "move". 1088 */ 1089 static bool mem_cgroup_under_move(struct mem_cgroup *memcg) 1090 { 1091 struct mem_cgroup *from; 1092 struct mem_cgroup *to; 1093 bool ret = false; 1094 /* 1095 * Unlike task_move routines, we access mc.to, mc.from not under 1096 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead. 1097 */ 1098 spin_lock(&mc.lock); 1099 from = mc.from; 1100 to = mc.to; 1101 if (!from) 1102 goto unlock; 1103 1104 ret = mem_cgroup_is_descendant(from, memcg) || 1105 mem_cgroup_is_descendant(to, memcg); 1106 unlock: 1107 spin_unlock(&mc.lock); 1108 return ret; 1109 } 1110 1111 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg) 1112 { 1113 if (mc.moving_task && current != mc.moving_task) { 1114 if (mem_cgroup_under_move(memcg)) { 1115 DEFINE_WAIT(wait); 1116 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE); 1117 /* moving charge context might have finished. */ 1118 if (mc.moving_task) 1119 schedule(); 1120 finish_wait(&mc.waitq, &wait); 1121 return true; 1122 } 1123 } 1124 return false; 1125 } 1126 1127 unsigned int memcg1_stats[] = { 1128 MEMCG_CACHE, 1129 MEMCG_RSS, 1130 MEMCG_RSS_HUGE, 1131 NR_SHMEM, 1132 NR_FILE_MAPPED, 1133 NR_FILE_DIRTY, 1134 NR_WRITEBACK, 1135 MEMCG_SWAP, 1136 }; 1137 1138 static const char *const memcg1_stat_names[] = { 1139 "cache", 1140 "rss", 1141 "rss_huge", 1142 "shmem", 1143 "mapped_file", 1144 "dirty", 1145 "writeback", 1146 "swap", 1147 }; 1148 1149 #define K(x) ((x) << (PAGE_SHIFT-10)) 1150 /** 1151 * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller. 1152 * @memcg: The memory cgroup that went over limit 1153 * @p: Task that is going to be killed 1154 * 1155 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is 1156 * enabled 1157 */ 1158 void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p) 1159 { 1160 struct mem_cgroup *iter; 1161 unsigned int i; 1162 1163 rcu_read_lock(); 1164 1165 if (p) { 1166 pr_info("Task in "); 1167 pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id)); 1168 pr_cont(" killed as a result of limit of "); 1169 } else { 1170 pr_info("Memory limit reached of cgroup "); 1171 } 1172 1173 pr_cont_cgroup_path(memcg->css.cgroup); 1174 pr_cont("\n"); 1175 1176 rcu_read_unlock(); 1177 1178 pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n", 1179 K((u64)page_counter_read(&memcg->memory)), 1180 K((u64)memcg->memory.limit), memcg->memory.failcnt); 1181 pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n", 1182 K((u64)page_counter_read(&memcg->memsw)), 1183 K((u64)memcg->memsw.limit), memcg->memsw.failcnt); 1184 pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n", 1185 K((u64)page_counter_read(&memcg->kmem)), 1186 K((u64)memcg->kmem.limit), memcg->kmem.failcnt); 1187 1188 for_each_mem_cgroup_tree(iter, memcg) { 1189 pr_info("Memory cgroup stats for "); 1190 pr_cont_cgroup_path(iter->css.cgroup); 1191 pr_cont(":"); 1192 1193 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) { 1194 if (memcg1_stats[i] == MEMCG_SWAP && !do_swap_account) 1195 continue; 1196 pr_cont(" %s:%luKB", memcg1_stat_names[i], 1197 K(memcg_page_state(iter, memcg1_stats[i]))); 1198 } 1199 1200 for (i = 0; i < NR_LRU_LISTS; i++) 1201 pr_cont(" %s:%luKB", mem_cgroup_lru_names[i], 1202 K(mem_cgroup_nr_lru_pages(iter, BIT(i)))); 1203 1204 pr_cont("\n"); 1205 } 1206 } 1207 1208 /* 1209 * This function returns the number of memcg under hierarchy tree. Returns 1210 * 1(self count) if no children. 1211 */ 1212 static int mem_cgroup_count_children(struct mem_cgroup *memcg) 1213 { 1214 int num = 0; 1215 struct mem_cgroup *iter; 1216 1217 for_each_mem_cgroup_tree(iter, memcg) 1218 num++; 1219 return num; 1220 } 1221 1222 /* 1223 * Return the memory (and swap, if configured) limit for a memcg. 1224 */ 1225 unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg) 1226 { 1227 unsigned long limit; 1228 1229 limit = memcg->memory.limit; 1230 if (mem_cgroup_swappiness(memcg)) { 1231 unsigned long memsw_limit; 1232 unsigned long swap_limit; 1233 1234 memsw_limit = memcg->memsw.limit; 1235 swap_limit = memcg->swap.limit; 1236 swap_limit = min(swap_limit, (unsigned long)total_swap_pages); 1237 limit = min(limit + swap_limit, memsw_limit); 1238 } 1239 return limit; 1240 } 1241 1242 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask, 1243 int order) 1244 { 1245 struct oom_control oc = { 1246 .zonelist = NULL, 1247 .nodemask = NULL, 1248 .memcg = memcg, 1249 .gfp_mask = gfp_mask, 1250 .order = order, 1251 }; 1252 bool ret; 1253 1254 mutex_lock(&oom_lock); 1255 ret = out_of_memory(&oc); 1256 mutex_unlock(&oom_lock); 1257 return ret; 1258 } 1259 1260 #if MAX_NUMNODES > 1 1261 1262 /** 1263 * test_mem_cgroup_node_reclaimable 1264 * @memcg: the target memcg 1265 * @nid: the node ID to be checked. 1266 * @noswap : specify true here if the user wants flle only information. 1267 * 1268 * This function returns whether the specified memcg contains any 1269 * reclaimable pages on a node. Returns true if there are any reclaimable 1270 * pages in the node. 1271 */ 1272 static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg, 1273 int nid, bool noswap) 1274 { 1275 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE)) 1276 return true; 1277 if (noswap || !total_swap_pages) 1278 return false; 1279 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON)) 1280 return true; 1281 return false; 1282 1283 } 1284 1285 /* 1286 * Always updating the nodemask is not very good - even if we have an empty 1287 * list or the wrong list here, we can start from some node and traverse all 1288 * nodes based on the zonelist. So update the list loosely once per 10 secs. 1289 * 1290 */ 1291 static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg) 1292 { 1293 int nid; 1294 /* 1295 * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET 1296 * pagein/pageout changes since the last update. 1297 */ 1298 if (!atomic_read(&memcg->numainfo_events)) 1299 return; 1300 if (atomic_inc_return(&memcg->numainfo_updating) > 1) 1301 return; 1302 1303 /* make a nodemask where this memcg uses memory from */ 1304 memcg->scan_nodes = node_states[N_MEMORY]; 1305 1306 for_each_node_mask(nid, node_states[N_MEMORY]) { 1307 1308 if (!test_mem_cgroup_node_reclaimable(memcg, nid, false)) 1309 node_clear(nid, memcg->scan_nodes); 1310 } 1311 1312 atomic_set(&memcg->numainfo_events, 0); 1313 atomic_set(&memcg->numainfo_updating, 0); 1314 } 1315 1316 /* 1317 * Selecting a node where we start reclaim from. Because what we need is just 1318 * reducing usage counter, start from anywhere is O,K. Considering 1319 * memory reclaim from current node, there are pros. and cons. 1320 * 1321 * Freeing memory from current node means freeing memory from a node which 1322 * we'll use or we've used. So, it may make LRU bad. And if several threads 1323 * hit limits, it will see a contention on a node. But freeing from remote 1324 * node means more costs for memory reclaim because of memory latency. 1325 * 1326 * Now, we use round-robin. Better algorithm is welcomed. 1327 */ 1328 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg) 1329 { 1330 int node; 1331 1332 mem_cgroup_may_update_nodemask(memcg); 1333 node = memcg->last_scanned_node; 1334 1335 node = next_node_in(node, memcg->scan_nodes); 1336 /* 1337 * mem_cgroup_may_update_nodemask might have seen no reclaimmable pages 1338 * last time it really checked all the LRUs due to rate limiting. 1339 * Fallback to the current node in that case for simplicity. 1340 */ 1341 if (unlikely(node == MAX_NUMNODES)) 1342 node = numa_node_id(); 1343 1344 memcg->last_scanned_node = node; 1345 return node; 1346 } 1347 #else 1348 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg) 1349 { 1350 return 0; 1351 } 1352 #endif 1353 1354 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg, 1355 pg_data_t *pgdat, 1356 gfp_t gfp_mask, 1357 unsigned long *total_scanned) 1358 { 1359 struct mem_cgroup *victim = NULL; 1360 int total = 0; 1361 int loop = 0; 1362 unsigned long excess; 1363 unsigned long nr_scanned; 1364 struct mem_cgroup_reclaim_cookie reclaim = { 1365 .pgdat = pgdat, 1366 .priority = 0, 1367 }; 1368 1369 excess = soft_limit_excess(root_memcg); 1370 1371 while (1) { 1372 victim = mem_cgroup_iter(root_memcg, victim, &reclaim); 1373 if (!victim) { 1374 loop++; 1375 if (loop >= 2) { 1376 /* 1377 * If we have not been able to reclaim 1378 * anything, it might because there are 1379 * no reclaimable pages under this hierarchy 1380 */ 1381 if (!total) 1382 break; 1383 /* 1384 * We want to do more targeted reclaim. 1385 * excess >> 2 is not to excessive so as to 1386 * reclaim too much, nor too less that we keep 1387 * coming back to reclaim from this cgroup 1388 */ 1389 if (total >= (excess >> 2) || 1390 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) 1391 break; 1392 } 1393 continue; 1394 } 1395 total += mem_cgroup_shrink_node(victim, gfp_mask, false, 1396 pgdat, &nr_scanned); 1397 *total_scanned += nr_scanned; 1398 if (!soft_limit_excess(root_memcg)) 1399 break; 1400 } 1401 mem_cgroup_iter_break(root_memcg, victim); 1402 return total; 1403 } 1404 1405 #ifdef CONFIG_LOCKDEP 1406 static struct lockdep_map memcg_oom_lock_dep_map = { 1407 .name = "memcg_oom_lock", 1408 }; 1409 #endif 1410 1411 static DEFINE_SPINLOCK(memcg_oom_lock); 1412 1413 /* 1414 * Check OOM-Killer is already running under our hierarchy. 1415 * If someone is running, return false. 1416 */ 1417 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg) 1418 { 1419 struct mem_cgroup *iter, *failed = NULL; 1420 1421 spin_lock(&memcg_oom_lock); 1422 1423 for_each_mem_cgroup_tree(iter, memcg) { 1424 if (iter->oom_lock) { 1425 /* 1426 * this subtree of our hierarchy is already locked 1427 * so we cannot give a lock. 1428 */ 1429 failed = iter; 1430 mem_cgroup_iter_break(memcg, iter); 1431 break; 1432 } else 1433 iter->oom_lock = true; 1434 } 1435 1436 if (failed) { 1437 /* 1438 * OK, we failed to lock the whole subtree so we have 1439 * to clean up what we set up to the failing subtree 1440 */ 1441 for_each_mem_cgroup_tree(iter, memcg) { 1442 if (iter == failed) { 1443 mem_cgroup_iter_break(memcg, iter); 1444 break; 1445 } 1446 iter->oom_lock = false; 1447 } 1448 } else 1449 mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_); 1450 1451 spin_unlock(&memcg_oom_lock); 1452 1453 return !failed; 1454 } 1455 1456 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg) 1457 { 1458 struct mem_cgroup *iter; 1459 1460 spin_lock(&memcg_oom_lock); 1461 mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_); 1462 for_each_mem_cgroup_tree(iter, memcg) 1463 iter->oom_lock = false; 1464 spin_unlock(&memcg_oom_lock); 1465 } 1466 1467 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg) 1468 { 1469 struct mem_cgroup *iter; 1470 1471 spin_lock(&memcg_oom_lock); 1472 for_each_mem_cgroup_tree(iter, memcg) 1473 iter->under_oom++; 1474 spin_unlock(&memcg_oom_lock); 1475 } 1476 1477 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg) 1478 { 1479 struct mem_cgroup *iter; 1480 1481 /* 1482 * When a new child is created while the hierarchy is under oom, 1483 * mem_cgroup_oom_lock() may not be called. Watch for underflow. 1484 */ 1485 spin_lock(&memcg_oom_lock); 1486 for_each_mem_cgroup_tree(iter, memcg) 1487 if (iter->under_oom > 0) 1488 iter->under_oom--; 1489 spin_unlock(&memcg_oom_lock); 1490 } 1491 1492 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq); 1493 1494 struct oom_wait_info { 1495 struct mem_cgroup *memcg; 1496 wait_queue_entry_t wait; 1497 }; 1498 1499 static int memcg_oom_wake_function(wait_queue_entry_t *wait, 1500 unsigned mode, int sync, void *arg) 1501 { 1502 struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg; 1503 struct mem_cgroup *oom_wait_memcg; 1504 struct oom_wait_info *oom_wait_info; 1505 1506 oom_wait_info = container_of(wait, struct oom_wait_info, wait); 1507 oom_wait_memcg = oom_wait_info->memcg; 1508 1509 if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) && 1510 !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg)) 1511 return 0; 1512 return autoremove_wake_function(wait, mode, sync, arg); 1513 } 1514 1515 static void memcg_oom_recover(struct mem_cgroup *memcg) 1516 { 1517 /* 1518 * For the following lockless ->under_oom test, the only required 1519 * guarantee is that it must see the state asserted by an OOM when 1520 * this function is called as a result of userland actions 1521 * triggered by the notification of the OOM. This is trivially 1522 * achieved by invoking mem_cgroup_mark_under_oom() before 1523 * triggering notification. 1524 */ 1525 if (memcg && memcg->under_oom) 1526 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg); 1527 } 1528 1529 static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order) 1530 { 1531 if (!current->memcg_may_oom) 1532 return; 1533 /* 1534 * We are in the middle of the charge context here, so we 1535 * don't want to block when potentially sitting on a callstack 1536 * that holds all kinds of filesystem and mm locks. 1537 * 1538 * Also, the caller may handle a failed allocation gracefully 1539 * (like optional page cache readahead) and so an OOM killer 1540 * invocation might not even be necessary. 1541 * 1542 * That's why we don't do anything here except remember the 1543 * OOM context and then deal with it at the end of the page 1544 * fault when the stack is unwound, the locks are released, 1545 * and when we know whether the fault was overall successful. 1546 */ 1547 css_get(&memcg->css); 1548 current->memcg_in_oom = memcg; 1549 current->memcg_oom_gfp_mask = mask; 1550 current->memcg_oom_order = order; 1551 } 1552 1553 /** 1554 * mem_cgroup_oom_synchronize - complete memcg OOM handling 1555 * @handle: actually kill/wait or just clean up the OOM state 1556 * 1557 * This has to be called at the end of a page fault if the memcg OOM 1558 * handler was enabled. 1559 * 1560 * Memcg supports userspace OOM handling where failed allocations must 1561 * sleep on a waitqueue until the userspace task resolves the 1562 * situation. Sleeping directly in the charge context with all kinds 1563 * of locks held is not a good idea, instead we remember an OOM state 1564 * in the task and mem_cgroup_oom_synchronize() has to be called at 1565 * the end of the page fault to complete the OOM handling. 1566 * 1567 * Returns %true if an ongoing memcg OOM situation was detected and 1568 * completed, %false otherwise. 1569 */ 1570 bool mem_cgroup_oom_synchronize(bool handle) 1571 { 1572 struct mem_cgroup *memcg = current->memcg_in_oom; 1573 struct oom_wait_info owait; 1574 bool locked; 1575 1576 /* OOM is global, do not handle */ 1577 if (!memcg) 1578 return false; 1579 1580 if (!handle) 1581 goto cleanup; 1582 1583 owait.memcg = memcg; 1584 owait.wait.flags = 0; 1585 owait.wait.func = memcg_oom_wake_function; 1586 owait.wait.private = current; 1587 INIT_LIST_HEAD(&owait.wait.entry); 1588 1589 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE); 1590 mem_cgroup_mark_under_oom(memcg); 1591 1592 locked = mem_cgroup_oom_trylock(memcg); 1593 1594 if (locked) 1595 mem_cgroup_oom_notify(memcg); 1596 1597 if (locked && !memcg->oom_kill_disable) { 1598 mem_cgroup_unmark_under_oom(memcg); 1599 finish_wait(&memcg_oom_waitq, &owait.wait); 1600 mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask, 1601 current->memcg_oom_order); 1602 } else { 1603 schedule(); 1604 mem_cgroup_unmark_under_oom(memcg); 1605 finish_wait(&memcg_oom_waitq, &owait.wait); 1606 } 1607 1608 if (locked) { 1609 mem_cgroup_oom_unlock(memcg); 1610 /* 1611 * There is no guarantee that an OOM-lock contender 1612 * sees the wakeups triggered by the OOM kill 1613 * uncharges. Wake any sleepers explicitely. 1614 */ 1615 memcg_oom_recover(memcg); 1616 } 1617 cleanup: 1618 current->memcg_in_oom = NULL; 1619 css_put(&memcg->css); 1620 return true; 1621 } 1622 1623 /** 1624 * lock_page_memcg - lock a page->mem_cgroup binding 1625 * @page: the page 1626 * 1627 * This function protects unlocked LRU pages from being moved to 1628 * another cgroup. 1629 * 1630 * It ensures lifetime of the returned memcg. Caller is responsible 1631 * for the lifetime of the page; __unlock_page_memcg() is available 1632 * when @page might get freed inside the locked section. 1633 */ 1634 struct mem_cgroup *lock_page_memcg(struct page *page) 1635 { 1636 struct mem_cgroup *memcg; 1637 unsigned long flags; 1638 1639 /* 1640 * The RCU lock is held throughout the transaction. The fast 1641 * path can get away without acquiring the memcg->move_lock 1642 * because page moving starts with an RCU grace period. 1643 * 1644 * The RCU lock also protects the memcg from being freed when 1645 * the page state that is going to change is the only thing 1646 * preventing the page itself from being freed. E.g. writeback 1647 * doesn't hold a page reference and relies on PG_writeback to 1648 * keep off truncation, migration and so forth. 1649 */ 1650 rcu_read_lock(); 1651 1652 if (mem_cgroup_disabled()) 1653 return NULL; 1654 again: 1655 memcg = page->mem_cgroup; 1656 if (unlikely(!memcg)) 1657 return NULL; 1658 1659 if (atomic_read(&memcg->moving_account) <= 0) 1660 return memcg; 1661 1662 spin_lock_irqsave(&memcg->move_lock, flags); 1663 if (memcg != page->mem_cgroup) { 1664 spin_unlock_irqrestore(&memcg->move_lock, flags); 1665 goto again; 1666 } 1667 1668 /* 1669 * When charge migration first begins, we can have locked and 1670 * unlocked page stat updates happening concurrently. Track 1671 * the task who has the lock for unlock_page_memcg(). 1672 */ 1673 memcg->move_lock_task = current; 1674 memcg->move_lock_flags = flags; 1675 1676 return memcg; 1677 } 1678 EXPORT_SYMBOL(lock_page_memcg); 1679 1680 /** 1681 * __unlock_page_memcg - unlock and unpin a memcg 1682 * @memcg: the memcg 1683 * 1684 * Unlock and unpin a memcg returned by lock_page_memcg(). 1685 */ 1686 void __unlock_page_memcg(struct mem_cgroup *memcg) 1687 { 1688 if (memcg && memcg->move_lock_task == current) { 1689 unsigned long flags = memcg->move_lock_flags; 1690 1691 memcg->move_lock_task = NULL; 1692 memcg->move_lock_flags = 0; 1693 1694 spin_unlock_irqrestore(&memcg->move_lock, flags); 1695 } 1696 1697 rcu_read_unlock(); 1698 } 1699 1700 /** 1701 * unlock_page_memcg - unlock a page->mem_cgroup binding 1702 * @page: the page 1703 */ 1704 void unlock_page_memcg(struct page *page) 1705 { 1706 __unlock_page_memcg(page->mem_cgroup); 1707 } 1708 EXPORT_SYMBOL(unlock_page_memcg); 1709 1710 /* 1711 * size of first charge trial. "32" comes from vmscan.c's magic value. 1712 * TODO: maybe necessary to use big numbers in big irons. 1713 */ 1714 #define CHARGE_BATCH 32U 1715 struct memcg_stock_pcp { 1716 struct mem_cgroup *cached; /* this never be root cgroup */ 1717 unsigned int nr_pages; 1718 struct work_struct work; 1719 unsigned long flags; 1720 #define FLUSHING_CACHED_CHARGE 0 1721 }; 1722 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock); 1723 static DEFINE_MUTEX(percpu_charge_mutex); 1724 1725 /** 1726 * consume_stock: Try to consume stocked charge on this cpu. 1727 * @memcg: memcg to consume from. 1728 * @nr_pages: how many pages to charge. 1729 * 1730 * The charges will only happen if @memcg matches the current cpu's memcg 1731 * stock, and at least @nr_pages are available in that stock. Failure to 1732 * service an allocation will refill the stock. 1733 * 1734 * returns true if successful, false otherwise. 1735 */ 1736 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) 1737 { 1738 struct memcg_stock_pcp *stock; 1739 unsigned long flags; 1740 bool ret = false; 1741 1742 if (nr_pages > CHARGE_BATCH) 1743 return ret; 1744 1745 local_irq_save(flags); 1746 1747 stock = this_cpu_ptr(&memcg_stock); 1748 if (memcg == stock->cached && stock->nr_pages >= nr_pages) { 1749 stock->nr_pages -= nr_pages; 1750 ret = true; 1751 } 1752 1753 local_irq_restore(flags); 1754 1755 return ret; 1756 } 1757 1758 /* 1759 * Returns stocks cached in percpu and reset cached information. 1760 */ 1761 static void drain_stock(struct memcg_stock_pcp *stock) 1762 { 1763 struct mem_cgroup *old = stock->cached; 1764 1765 if (stock->nr_pages) { 1766 page_counter_uncharge(&old->memory, stock->nr_pages); 1767 if (do_memsw_account()) 1768 page_counter_uncharge(&old->memsw, stock->nr_pages); 1769 css_put_many(&old->css, stock->nr_pages); 1770 stock->nr_pages = 0; 1771 } 1772 stock->cached = NULL; 1773 } 1774 1775 static void drain_local_stock(struct work_struct *dummy) 1776 { 1777 struct memcg_stock_pcp *stock; 1778 unsigned long flags; 1779 1780 local_irq_save(flags); 1781 1782 stock = this_cpu_ptr(&memcg_stock); 1783 drain_stock(stock); 1784 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags); 1785 1786 local_irq_restore(flags); 1787 } 1788 1789 /* 1790 * Cache charges(val) to local per_cpu area. 1791 * This will be consumed by consume_stock() function, later. 1792 */ 1793 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) 1794 { 1795 struct memcg_stock_pcp *stock; 1796 unsigned long flags; 1797 1798 local_irq_save(flags); 1799 1800 stock = this_cpu_ptr(&memcg_stock); 1801 if (stock->cached != memcg) { /* reset if necessary */ 1802 drain_stock(stock); 1803 stock->cached = memcg; 1804 } 1805 stock->nr_pages += nr_pages; 1806 1807 if (stock->nr_pages > CHARGE_BATCH) 1808 drain_stock(stock); 1809 1810 local_irq_restore(flags); 1811 } 1812 1813 /* 1814 * Drains all per-CPU charge caches for given root_memcg resp. subtree 1815 * of the hierarchy under it. 1816 */ 1817 static void drain_all_stock(struct mem_cgroup *root_memcg) 1818 { 1819 int cpu, curcpu; 1820 1821 /* If someone's already draining, avoid adding running more workers. */ 1822 if (!mutex_trylock(&percpu_charge_mutex)) 1823 return; 1824 /* Notify other cpus that system-wide "drain" is running */ 1825 get_online_cpus(); 1826 curcpu = get_cpu(); 1827 for_each_online_cpu(cpu) { 1828 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); 1829 struct mem_cgroup *memcg; 1830 1831 memcg = stock->cached; 1832 if (!memcg || !stock->nr_pages) 1833 continue; 1834 if (!mem_cgroup_is_descendant(memcg, root_memcg)) 1835 continue; 1836 if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) { 1837 if (cpu == curcpu) 1838 drain_local_stock(&stock->work); 1839 else 1840 schedule_work_on(cpu, &stock->work); 1841 } 1842 } 1843 put_cpu(); 1844 put_online_cpus(); 1845 mutex_unlock(&percpu_charge_mutex); 1846 } 1847 1848 static int memcg_hotplug_cpu_dead(unsigned int cpu) 1849 { 1850 struct memcg_stock_pcp *stock; 1851 1852 stock = &per_cpu(memcg_stock, cpu); 1853 drain_stock(stock); 1854 return 0; 1855 } 1856 1857 static void reclaim_high(struct mem_cgroup *memcg, 1858 unsigned int nr_pages, 1859 gfp_t gfp_mask) 1860 { 1861 do { 1862 if (page_counter_read(&memcg->memory) <= memcg->high) 1863 continue; 1864 mem_cgroup_event(memcg, MEMCG_HIGH); 1865 try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true); 1866 } while ((memcg = parent_mem_cgroup(memcg))); 1867 } 1868 1869 static void high_work_func(struct work_struct *work) 1870 { 1871 struct mem_cgroup *memcg; 1872 1873 memcg = container_of(work, struct mem_cgroup, high_work); 1874 reclaim_high(memcg, CHARGE_BATCH, GFP_KERNEL); 1875 } 1876 1877 /* 1878 * Scheduled by try_charge() to be executed from the userland return path 1879 * and reclaims memory over the high limit. 1880 */ 1881 void mem_cgroup_handle_over_high(void) 1882 { 1883 unsigned int nr_pages = current->memcg_nr_pages_over_high; 1884 struct mem_cgroup *memcg; 1885 1886 if (likely(!nr_pages)) 1887 return; 1888 1889 memcg = get_mem_cgroup_from_mm(current->mm); 1890 reclaim_high(memcg, nr_pages, GFP_KERNEL); 1891 css_put(&memcg->css); 1892 current->memcg_nr_pages_over_high = 0; 1893 } 1894 1895 static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, 1896 unsigned int nr_pages) 1897 { 1898 unsigned int batch = max(CHARGE_BATCH, nr_pages); 1899 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES; 1900 struct mem_cgroup *mem_over_limit; 1901 struct page_counter *counter; 1902 unsigned long nr_reclaimed; 1903 bool may_swap = true; 1904 bool drained = false; 1905 1906 if (mem_cgroup_is_root(memcg)) 1907 return 0; 1908 retry: 1909 if (consume_stock(memcg, nr_pages)) 1910 return 0; 1911 1912 if (!do_memsw_account() || 1913 page_counter_try_charge(&memcg->memsw, batch, &counter)) { 1914 if (page_counter_try_charge(&memcg->memory, batch, &counter)) 1915 goto done_restock; 1916 if (do_memsw_account()) 1917 page_counter_uncharge(&memcg->memsw, batch); 1918 mem_over_limit = mem_cgroup_from_counter(counter, memory); 1919 } else { 1920 mem_over_limit = mem_cgroup_from_counter(counter, memsw); 1921 may_swap = false; 1922 } 1923 1924 if (batch > nr_pages) { 1925 batch = nr_pages; 1926 goto retry; 1927 } 1928 1929 /* 1930 * Unlike in global OOM situations, memcg is not in a physical 1931 * memory shortage. Allow dying and OOM-killed tasks to 1932 * bypass the last charges so that they can exit quickly and 1933 * free their memory. 1934 */ 1935 if (unlikely(tsk_is_oom_victim(current) || 1936 fatal_signal_pending(current) || 1937 current->flags & PF_EXITING)) 1938 goto force; 1939 1940 /* 1941 * Prevent unbounded recursion when reclaim operations need to 1942 * allocate memory. This might exceed the limits temporarily, 1943 * but we prefer facilitating memory reclaim and getting back 1944 * under the limit over triggering OOM kills in these cases. 1945 */ 1946 if (unlikely(current->flags & PF_MEMALLOC)) 1947 goto force; 1948 1949 if (unlikely(task_in_memcg_oom(current))) 1950 goto nomem; 1951 1952 if (!gfpflags_allow_blocking(gfp_mask)) 1953 goto nomem; 1954 1955 mem_cgroup_event(mem_over_limit, MEMCG_MAX); 1956 1957 nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages, 1958 gfp_mask, may_swap); 1959 1960 if (mem_cgroup_margin(mem_over_limit) >= nr_pages) 1961 goto retry; 1962 1963 if (!drained) { 1964 drain_all_stock(mem_over_limit); 1965 drained = true; 1966 goto retry; 1967 } 1968 1969 if (gfp_mask & __GFP_NORETRY) 1970 goto nomem; 1971 /* 1972 * Even though the limit is exceeded at this point, reclaim 1973 * may have been able to free some pages. Retry the charge 1974 * before killing the task. 1975 * 1976 * Only for regular pages, though: huge pages are rather 1977 * unlikely to succeed so close to the limit, and we fall back 1978 * to regular pages anyway in case of failure. 1979 */ 1980 if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER)) 1981 goto retry; 1982 /* 1983 * At task move, charge accounts can be doubly counted. So, it's 1984 * better to wait until the end of task_move if something is going on. 1985 */ 1986 if (mem_cgroup_wait_acct_move(mem_over_limit)) 1987 goto retry; 1988 1989 if (nr_retries--) 1990 goto retry; 1991 1992 if (gfp_mask & __GFP_NOFAIL) 1993 goto force; 1994 1995 if (fatal_signal_pending(current)) 1996 goto force; 1997 1998 mem_cgroup_event(mem_over_limit, MEMCG_OOM); 1999 2000 mem_cgroup_oom(mem_over_limit, gfp_mask, 2001 get_order(nr_pages * PAGE_SIZE)); 2002 nomem: 2003 if (!(gfp_mask & __GFP_NOFAIL)) 2004 return -ENOMEM; 2005 force: 2006 /* 2007 * The allocation either can't fail or will lead to more memory 2008 * being freed very soon. Allow memory usage go over the limit 2009 * temporarily by force charging it. 2010 */ 2011 page_counter_charge(&memcg->memory, nr_pages); 2012 if (do_memsw_account()) 2013 page_counter_charge(&memcg->memsw, nr_pages); 2014 css_get_many(&memcg->css, nr_pages); 2015 2016 return 0; 2017 2018 done_restock: 2019 css_get_many(&memcg->css, batch); 2020 if (batch > nr_pages) 2021 refill_stock(memcg, batch - nr_pages); 2022 2023 /* 2024 * If the hierarchy is above the normal consumption range, schedule 2025 * reclaim on returning to userland. We can perform reclaim here 2026 * if __GFP_RECLAIM but let's always punt for simplicity and so that 2027 * GFP_KERNEL can consistently be used during reclaim. @memcg is 2028 * not recorded as it most likely matches current's and won't 2029 * change in the meantime. As high limit is checked again before 2030 * reclaim, the cost of mismatch is negligible. 2031 */ 2032 do { 2033 if (page_counter_read(&memcg->memory) > memcg->high) { 2034 /* Don't bother a random interrupted task */ 2035 if (in_interrupt()) { 2036 schedule_work(&memcg->high_work); 2037 break; 2038 } 2039 current->memcg_nr_pages_over_high += batch; 2040 set_notify_resume(current); 2041 break; 2042 } 2043 } while ((memcg = parent_mem_cgroup(memcg))); 2044 2045 return 0; 2046 } 2047 2048 static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages) 2049 { 2050 if (mem_cgroup_is_root(memcg)) 2051 return; 2052 2053 page_counter_uncharge(&memcg->memory, nr_pages); 2054 if (do_memsw_account()) 2055 page_counter_uncharge(&memcg->memsw, nr_pages); 2056 2057 css_put_many(&memcg->css, nr_pages); 2058 } 2059 2060 static void lock_page_lru(struct page *page, int *isolated) 2061 { 2062 struct zone *zone = page_zone(page); 2063 2064 spin_lock_irq(zone_lru_lock(zone)); 2065 if (PageLRU(page)) { 2066 struct lruvec *lruvec; 2067 2068 lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat); 2069 ClearPageLRU(page); 2070 del_page_from_lru_list(page, lruvec, page_lru(page)); 2071 *isolated = 1; 2072 } else 2073 *isolated = 0; 2074 } 2075 2076 static void unlock_page_lru(struct page *page, int isolated) 2077 { 2078 struct zone *zone = page_zone(page); 2079 2080 if (isolated) { 2081 struct lruvec *lruvec; 2082 2083 lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat); 2084 VM_BUG_ON_PAGE(PageLRU(page), page); 2085 SetPageLRU(page); 2086 add_page_to_lru_list(page, lruvec, page_lru(page)); 2087 } 2088 spin_unlock_irq(zone_lru_lock(zone)); 2089 } 2090 2091 static void commit_charge(struct page *page, struct mem_cgroup *memcg, 2092 bool lrucare) 2093 { 2094 int isolated; 2095 2096 VM_BUG_ON_PAGE(page->mem_cgroup, page); 2097 2098 /* 2099 * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page 2100 * may already be on some other mem_cgroup's LRU. Take care of it. 2101 */ 2102 if (lrucare) 2103 lock_page_lru(page, &isolated); 2104 2105 /* 2106 * Nobody should be changing or seriously looking at 2107 * page->mem_cgroup at this point: 2108 * 2109 * - the page is uncharged 2110 * 2111 * - the page is off-LRU 2112 * 2113 * - an anonymous fault has exclusive page access, except for 2114 * a locked page table 2115 * 2116 * - a page cache insertion, a swapin fault, or a migration 2117 * have the page locked 2118 */ 2119 page->mem_cgroup = memcg; 2120 2121 if (lrucare) 2122 unlock_page_lru(page, isolated); 2123 } 2124 2125 #ifndef CONFIG_SLOB 2126 static int memcg_alloc_cache_id(void) 2127 { 2128 int id, size; 2129 int err; 2130 2131 id = ida_simple_get(&memcg_cache_ida, 2132 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL); 2133 if (id < 0) 2134 return id; 2135 2136 if (id < memcg_nr_cache_ids) 2137 return id; 2138 2139 /* 2140 * There's no space for the new id in memcg_caches arrays, 2141 * so we have to grow them. 2142 */ 2143 down_write(&memcg_cache_ids_sem); 2144 2145 size = 2 * (id + 1); 2146 if (size < MEMCG_CACHES_MIN_SIZE) 2147 size = MEMCG_CACHES_MIN_SIZE; 2148 else if (size > MEMCG_CACHES_MAX_SIZE) 2149 size = MEMCG_CACHES_MAX_SIZE; 2150 2151 err = memcg_update_all_caches(size); 2152 if (!err) 2153 err = memcg_update_all_list_lrus(size); 2154 if (!err) 2155 memcg_nr_cache_ids = size; 2156 2157 up_write(&memcg_cache_ids_sem); 2158 2159 if (err) { 2160 ida_simple_remove(&memcg_cache_ida, id); 2161 return err; 2162 } 2163 return id; 2164 } 2165 2166 static void memcg_free_cache_id(int id) 2167 { 2168 ida_simple_remove(&memcg_cache_ida, id); 2169 } 2170 2171 struct memcg_kmem_cache_create_work { 2172 struct mem_cgroup *memcg; 2173 struct kmem_cache *cachep; 2174 struct work_struct work; 2175 }; 2176 2177 static void memcg_kmem_cache_create_func(struct work_struct *w) 2178 { 2179 struct memcg_kmem_cache_create_work *cw = 2180 container_of(w, struct memcg_kmem_cache_create_work, work); 2181 struct mem_cgroup *memcg = cw->memcg; 2182 struct kmem_cache *cachep = cw->cachep; 2183 2184 memcg_create_kmem_cache(memcg, cachep); 2185 2186 css_put(&memcg->css); 2187 kfree(cw); 2188 } 2189 2190 /* 2191 * Enqueue the creation of a per-memcg kmem_cache. 2192 */ 2193 static void __memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg, 2194 struct kmem_cache *cachep) 2195 { 2196 struct memcg_kmem_cache_create_work *cw; 2197 2198 cw = kmalloc(sizeof(*cw), GFP_NOWAIT); 2199 if (!cw) 2200 return; 2201 2202 css_get(&memcg->css); 2203 2204 cw->memcg = memcg; 2205 cw->cachep = cachep; 2206 INIT_WORK(&cw->work, memcg_kmem_cache_create_func); 2207 2208 queue_work(memcg_kmem_cache_wq, &cw->work); 2209 } 2210 2211 static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg, 2212 struct kmem_cache *cachep) 2213 { 2214 /* 2215 * We need to stop accounting when we kmalloc, because if the 2216 * corresponding kmalloc cache is not yet created, the first allocation 2217 * in __memcg_schedule_kmem_cache_create will recurse. 2218 * 2219 * However, it is better to enclose the whole function. Depending on 2220 * the debugging options enabled, INIT_WORK(), for instance, can 2221 * trigger an allocation. This too, will make us recurse. Because at 2222 * this point we can't allow ourselves back into memcg_kmem_get_cache, 2223 * the safest choice is to do it like this, wrapping the whole function. 2224 */ 2225 current->memcg_kmem_skip_account = 1; 2226 __memcg_schedule_kmem_cache_create(memcg, cachep); 2227 current->memcg_kmem_skip_account = 0; 2228 } 2229 2230 static inline bool memcg_kmem_bypass(void) 2231 { 2232 if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD)) 2233 return true; 2234 return false; 2235 } 2236 2237 /** 2238 * memcg_kmem_get_cache: select the correct per-memcg cache for allocation 2239 * @cachep: the original global kmem cache 2240 * 2241 * Return the kmem_cache we're supposed to use for a slab allocation. 2242 * We try to use the current memcg's version of the cache. 2243 * 2244 * If the cache does not exist yet, if we are the first user of it, we 2245 * create it asynchronously in a workqueue and let the current allocation 2246 * go through with the original cache. 2247 * 2248 * This function takes a reference to the cache it returns to assure it 2249 * won't get destroyed while we are working with it. Once the caller is 2250 * done with it, memcg_kmem_put_cache() must be called to release the 2251 * reference. 2252 */ 2253 struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep) 2254 { 2255 struct mem_cgroup *memcg; 2256 struct kmem_cache *memcg_cachep; 2257 int kmemcg_id; 2258 2259 VM_BUG_ON(!is_root_cache(cachep)); 2260 2261 if (memcg_kmem_bypass()) 2262 return cachep; 2263 2264 if (current->memcg_kmem_skip_account) 2265 return cachep; 2266 2267 memcg = get_mem_cgroup_from_mm(current->mm); 2268 kmemcg_id = READ_ONCE(memcg->kmemcg_id); 2269 if (kmemcg_id < 0) 2270 goto out; 2271 2272 memcg_cachep = cache_from_memcg_idx(cachep, kmemcg_id); 2273 if (likely(memcg_cachep)) 2274 return memcg_cachep; 2275 2276 /* 2277 * If we are in a safe context (can wait, and not in interrupt 2278 * context), we could be be predictable and return right away. 2279 * This would guarantee that the allocation being performed 2280 * already belongs in the new cache. 2281 * 2282 * However, there are some clashes that can arrive from locking. 2283 * For instance, because we acquire the slab_mutex while doing 2284 * memcg_create_kmem_cache, this means no further allocation 2285 * could happen with the slab_mutex held. So it's better to 2286 * defer everything. 2287 */ 2288 memcg_schedule_kmem_cache_create(memcg, cachep); 2289 out: 2290 css_put(&memcg->css); 2291 return cachep; 2292 } 2293 2294 /** 2295 * memcg_kmem_put_cache: drop reference taken by memcg_kmem_get_cache 2296 * @cachep: the cache returned by memcg_kmem_get_cache 2297 */ 2298 void memcg_kmem_put_cache(struct kmem_cache *cachep) 2299 { 2300 if (!is_root_cache(cachep)) 2301 css_put(&cachep->memcg_params.memcg->css); 2302 } 2303 2304 /** 2305 * memcg_kmem_charge: charge a kmem page 2306 * @page: page to charge 2307 * @gfp: reclaim mode 2308 * @order: allocation order 2309 * @memcg: memory cgroup to charge 2310 * 2311 * Returns 0 on success, an error code on failure. 2312 */ 2313 int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order, 2314 struct mem_cgroup *memcg) 2315 { 2316 unsigned int nr_pages = 1 << order; 2317 struct page_counter *counter; 2318 int ret; 2319 2320 ret = try_charge(memcg, gfp, nr_pages); 2321 if (ret) 2322 return ret; 2323 2324 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && 2325 !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) { 2326 cancel_charge(memcg, nr_pages); 2327 return -ENOMEM; 2328 } 2329 2330 page->mem_cgroup = memcg; 2331 2332 return 0; 2333 } 2334 2335 /** 2336 * memcg_kmem_charge: charge a kmem page to the current memory cgroup 2337 * @page: page to charge 2338 * @gfp: reclaim mode 2339 * @order: allocation order 2340 * 2341 * Returns 0 on success, an error code on failure. 2342 */ 2343 int memcg_kmem_charge(struct page *page, gfp_t gfp, int order) 2344 { 2345 struct mem_cgroup *memcg; 2346 int ret = 0; 2347 2348 if (memcg_kmem_bypass()) 2349 return 0; 2350 2351 memcg = get_mem_cgroup_from_mm(current->mm); 2352 if (!mem_cgroup_is_root(memcg)) { 2353 ret = memcg_kmem_charge_memcg(page, gfp, order, memcg); 2354 if (!ret) 2355 __SetPageKmemcg(page); 2356 } 2357 css_put(&memcg->css); 2358 return ret; 2359 } 2360 /** 2361 * memcg_kmem_uncharge: uncharge a kmem page 2362 * @page: page to uncharge 2363 * @order: allocation order 2364 */ 2365 void memcg_kmem_uncharge(struct page *page, int order) 2366 { 2367 struct mem_cgroup *memcg = page->mem_cgroup; 2368 unsigned int nr_pages = 1 << order; 2369 2370 if (!memcg) 2371 return; 2372 2373 VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page); 2374 2375 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) 2376 page_counter_uncharge(&memcg->kmem, nr_pages); 2377 2378 page_counter_uncharge(&memcg->memory, nr_pages); 2379 if (do_memsw_account()) 2380 page_counter_uncharge(&memcg->memsw, nr_pages); 2381 2382 page->mem_cgroup = NULL; 2383 2384 /* slab pages do not have PageKmemcg flag set */ 2385 if (PageKmemcg(page)) 2386 __ClearPageKmemcg(page); 2387 2388 css_put_many(&memcg->css, nr_pages); 2389 } 2390 #endif /* !CONFIG_SLOB */ 2391 2392 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2393 2394 /* 2395 * Because tail pages are not marked as "used", set it. We're under 2396 * zone_lru_lock and migration entries setup in all page mappings. 2397 */ 2398 void mem_cgroup_split_huge_fixup(struct page *head) 2399 { 2400 int i; 2401 2402 if (mem_cgroup_disabled()) 2403 return; 2404 2405 for (i = 1; i < HPAGE_PMD_NR; i++) 2406 head[i].mem_cgroup = head->mem_cgroup; 2407 2408 __this_cpu_sub(head->mem_cgroup->stat->count[MEMCG_RSS_HUGE], 2409 HPAGE_PMD_NR); 2410 } 2411 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 2412 2413 #ifdef CONFIG_MEMCG_SWAP 2414 static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg, 2415 int nr_entries) 2416 { 2417 this_cpu_add(memcg->stat->count[MEMCG_SWAP], nr_entries); 2418 } 2419 2420 /** 2421 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record. 2422 * @entry: swap entry to be moved 2423 * @from: mem_cgroup which the entry is moved from 2424 * @to: mem_cgroup which the entry is moved to 2425 * 2426 * It succeeds only when the swap_cgroup's record for this entry is the same 2427 * as the mem_cgroup's id of @from. 2428 * 2429 * Returns 0 on success, -EINVAL on failure. 2430 * 2431 * The caller must have charged to @to, IOW, called page_counter_charge() about 2432 * both res and memsw, and called css_get(). 2433 */ 2434 static int mem_cgroup_move_swap_account(swp_entry_t entry, 2435 struct mem_cgroup *from, struct mem_cgroup *to) 2436 { 2437 unsigned short old_id, new_id; 2438 2439 old_id = mem_cgroup_id(from); 2440 new_id = mem_cgroup_id(to); 2441 2442 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) { 2443 mem_cgroup_swap_statistics(from, -1); 2444 mem_cgroup_swap_statistics(to, 1); 2445 return 0; 2446 } 2447 return -EINVAL; 2448 } 2449 #else 2450 static inline int mem_cgroup_move_swap_account(swp_entry_t entry, 2451 struct mem_cgroup *from, struct mem_cgroup *to) 2452 { 2453 return -EINVAL; 2454 } 2455 #endif 2456 2457 static DEFINE_MUTEX(memcg_limit_mutex); 2458 2459 static int mem_cgroup_resize_limit(struct mem_cgroup *memcg, 2460 unsigned long limit) 2461 { 2462 unsigned long curusage; 2463 unsigned long oldusage; 2464 bool enlarge = false; 2465 int retry_count; 2466 int ret; 2467 2468 /* 2469 * For keeping hierarchical_reclaim simple, how long we should retry 2470 * is depends on callers. We set our retry-count to be function 2471 * of # of children which we should visit in this loop. 2472 */ 2473 retry_count = MEM_CGROUP_RECLAIM_RETRIES * 2474 mem_cgroup_count_children(memcg); 2475 2476 oldusage = page_counter_read(&memcg->memory); 2477 2478 do { 2479 if (signal_pending(current)) { 2480 ret = -EINTR; 2481 break; 2482 } 2483 2484 mutex_lock(&memcg_limit_mutex); 2485 if (limit > memcg->memsw.limit) { 2486 mutex_unlock(&memcg_limit_mutex); 2487 ret = -EINVAL; 2488 break; 2489 } 2490 if (limit > memcg->memory.limit) 2491 enlarge = true; 2492 ret = page_counter_limit(&memcg->memory, limit); 2493 mutex_unlock(&memcg_limit_mutex); 2494 2495 if (!ret) 2496 break; 2497 2498 try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, true); 2499 2500 curusage = page_counter_read(&memcg->memory); 2501 /* Usage is reduced ? */ 2502 if (curusage >= oldusage) 2503 retry_count--; 2504 else 2505 oldusage = curusage; 2506 } while (retry_count); 2507 2508 if (!ret && enlarge) 2509 memcg_oom_recover(memcg); 2510 2511 return ret; 2512 } 2513 2514 static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg, 2515 unsigned long limit) 2516 { 2517 unsigned long curusage; 2518 unsigned long oldusage; 2519 bool enlarge = false; 2520 int retry_count; 2521 int ret; 2522 2523 /* see mem_cgroup_resize_res_limit */ 2524 retry_count = MEM_CGROUP_RECLAIM_RETRIES * 2525 mem_cgroup_count_children(memcg); 2526 2527 oldusage = page_counter_read(&memcg->memsw); 2528 2529 do { 2530 if (signal_pending(current)) { 2531 ret = -EINTR; 2532 break; 2533 } 2534 2535 mutex_lock(&memcg_limit_mutex); 2536 if (limit < memcg->memory.limit) { 2537 mutex_unlock(&memcg_limit_mutex); 2538 ret = -EINVAL; 2539 break; 2540 } 2541 if (limit > memcg->memsw.limit) 2542 enlarge = true; 2543 ret = page_counter_limit(&memcg->memsw, limit); 2544 mutex_unlock(&memcg_limit_mutex); 2545 2546 if (!ret) 2547 break; 2548 2549 try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, false); 2550 2551 curusage = page_counter_read(&memcg->memsw); 2552 /* Usage is reduced ? */ 2553 if (curusage >= oldusage) 2554 retry_count--; 2555 else 2556 oldusage = curusage; 2557 } while (retry_count); 2558 2559 if (!ret && enlarge) 2560 memcg_oom_recover(memcg); 2561 2562 return ret; 2563 } 2564 2565 unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order, 2566 gfp_t gfp_mask, 2567 unsigned long *total_scanned) 2568 { 2569 unsigned long nr_reclaimed = 0; 2570 struct mem_cgroup_per_node *mz, *next_mz = NULL; 2571 unsigned long reclaimed; 2572 int loop = 0; 2573 struct mem_cgroup_tree_per_node *mctz; 2574 unsigned long excess; 2575 unsigned long nr_scanned; 2576 2577 if (order > 0) 2578 return 0; 2579 2580 mctz = soft_limit_tree_node(pgdat->node_id); 2581 2582 /* 2583 * Do not even bother to check the largest node if the root 2584 * is empty. Do it lockless to prevent lock bouncing. Races 2585 * are acceptable as soft limit is best effort anyway. 2586 */ 2587 if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root)) 2588 return 0; 2589 2590 /* 2591 * This loop can run a while, specially if mem_cgroup's continuously 2592 * keep exceeding their soft limit and putting the system under 2593 * pressure 2594 */ 2595 do { 2596 if (next_mz) 2597 mz = next_mz; 2598 else 2599 mz = mem_cgroup_largest_soft_limit_node(mctz); 2600 if (!mz) 2601 break; 2602 2603 nr_scanned = 0; 2604 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat, 2605 gfp_mask, &nr_scanned); 2606 nr_reclaimed += reclaimed; 2607 *total_scanned += nr_scanned; 2608 spin_lock_irq(&mctz->lock); 2609 __mem_cgroup_remove_exceeded(mz, mctz); 2610 2611 /* 2612 * If we failed to reclaim anything from this memory cgroup 2613 * it is time to move on to the next cgroup 2614 */ 2615 next_mz = NULL; 2616 if (!reclaimed) 2617 next_mz = __mem_cgroup_largest_soft_limit_node(mctz); 2618 2619 excess = soft_limit_excess(mz->memcg); 2620 /* 2621 * One school of thought says that we should not add 2622 * back the node to the tree if reclaim returns 0. 2623 * But our reclaim could return 0, simply because due 2624 * to priority we are exposing a smaller subset of 2625 * memory to reclaim from. Consider this as a longer 2626 * term TODO. 2627 */ 2628 /* If excess == 0, no tree ops */ 2629 __mem_cgroup_insert_exceeded(mz, mctz, excess); 2630 spin_unlock_irq(&mctz->lock); 2631 css_put(&mz->memcg->css); 2632 loop++; 2633 /* 2634 * Could not reclaim anything and there are no more 2635 * mem cgroups to try or we seem to be looping without 2636 * reclaiming anything. 2637 */ 2638 if (!nr_reclaimed && 2639 (next_mz == NULL || 2640 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS)) 2641 break; 2642 } while (!nr_reclaimed); 2643 if (next_mz) 2644 css_put(&next_mz->memcg->css); 2645 return nr_reclaimed; 2646 } 2647 2648 /* 2649 * Test whether @memcg has children, dead or alive. Note that this 2650 * function doesn't care whether @memcg has use_hierarchy enabled and 2651 * returns %true if there are child csses according to the cgroup 2652 * hierarchy. Testing use_hierarchy is the caller's responsiblity. 2653 */ 2654 static inline bool memcg_has_children(struct mem_cgroup *memcg) 2655 { 2656 bool ret; 2657 2658 rcu_read_lock(); 2659 ret = css_next_child(NULL, &memcg->css); 2660 rcu_read_unlock(); 2661 return ret; 2662 } 2663 2664 /* 2665 * Reclaims as many pages from the given memcg as possible. 2666 * 2667 * Caller is responsible for holding css reference for memcg. 2668 */ 2669 static int mem_cgroup_force_empty(struct mem_cgroup *memcg) 2670 { 2671 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES; 2672 2673 /* we call try-to-free pages for make this cgroup empty */ 2674 lru_add_drain_all(); 2675 /* try to free all pages in this cgroup */ 2676 while (nr_retries && page_counter_read(&memcg->memory)) { 2677 int progress; 2678 2679 if (signal_pending(current)) 2680 return -EINTR; 2681 2682 progress = try_to_free_mem_cgroup_pages(memcg, 1, 2683 GFP_KERNEL, true); 2684 if (!progress) { 2685 nr_retries--; 2686 /* maybe some writeback is necessary */ 2687 congestion_wait(BLK_RW_ASYNC, HZ/10); 2688 } 2689 2690 } 2691 2692 return 0; 2693 } 2694 2695 static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of, 2696 char *buf, size_t nbytes, 2697 loff_t off) 2698 { 2699 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 2700 2701 if (mem_cgroup_is_root(memcg)) 2702 return -EINVAL; 2703 return mem_cgroup_force_empty(memcg) ?: nbytes; 2704 } 2705 2706 static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css, 2707 struct cftype *cft) 2708 { 2709 return mem_cgroup_from_css(css)->use_hierarchy; 2710 } 2711 2712 static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css, 2713 struct cftype *cft, u64 val) 2714 { 2715 int retval = 0; 2716 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 2717 struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent); 2718 2719 if (memcg->use_hierarchy == val) 2720 return 0; 2721 2722 /* 2723 * If parent's use_hierarchy is set, we can't make any modifications 2724 * in the child subtrees. If it is unset, then the change can 2725 * occur, provided the current cgroup has no children. 2726 * 2727 * For the root cgroup, parent_mem is NULL, we allow value to be 2728 * set if there are no children. 2729 */ 2730 if ((!parent_memcg || !parent_memcg->use_hierarchy) && 2731 (val == 1 || val == 0)) { 2732 if (!memcg_has_children(memcg)) 2733 memcg->use_hierarchy = val; 2734 else 2735 retval = -EBUSY; 2736 } else 2737 retval = -EINVAL; 2738 2739 return retval; 2740 } 2741 2742 static void tree_stat(struct mem_cgroup *memcg, unsigned long *stat) 2743 { 2744 struct mem_cgroup *iter; 2745 int i; 2746 2747 memset(stat, 0, sizeof(*stat) * MEMCG_NR_STAT); 2748 2749 for_each_mem_cgroup_tree(iter, memcg) { 2750 for (i = 0; i < MEMCG_NR_STAT; i++) 2751 stat[i] += memcg_page_state(iter, i); 2752 } 2753 } 2754 2755 static void tree_events(struct mem_cgroup *memcg, unsigned long *events) 2756 { 2757 struct mem_cgroup *iter; 2758 int i; 2759 2760 memset(events, 0, sizeof(*events) * MEMCG_NR_EVENTS); 2761 2762 for_each_mem_cgroup_tree(iter, memcg) { 2763 for (i = 0; i < MEMCG_NR_EVENTS; i++) 2764 events[i] += memcg_sum_events(iter, i); 2765 } 2766 } 2767 2768 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap) 2769 { 2770 unsigned long val = 0; 2771 2772 if (mem_cgroup_is_root(memcg)) { 2773 struct mem_cgroup *iter; 2774 2775 for_each_mem_cgroup_tree(iter, memcg) { 2776 val += memcg_page_state(iter, MEMCG_CACHE); 2777 val += memcg_page_state(iter, MEMCG_RSS); 2778 if (swap) 2779 val += memcg_page_state(iter, MEMCG_SWAP); 2780 } 2781 } else { 2782 if (!swap) 2783 val = page_counter_read(&memcg->memory); 2784 else 2785 val = page_counter_read(&memcg->memsw); 2786 } 2787 return val; 2788 } 2789 2790 enum { 2791 RES_USAGE, 2792 RES_LIMIT, 2793 RES_MAX_USAGE, 2794 RES_FAILCNT, 2795 RES_SOFT_LIMIT, 2796 }; 2797 2798 static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css, 2799 struct cftype *cft) 2800 { 2801 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 2802 struct page_counter *counter; 2803 2804 switch (MEMFILE_TYPE(cft->private)) { 2805 case _MEM: 2806 counter = &memcg->memory; 2807 break; 2808 case _MEMSWAP: 2809 counter = &memcg->memsw; 2810 break; 2811 case _KMEM: 2812 counter = &memcg->kmem; 2813 break; 2814 case _TCP: 2815 counter = &memcg->tcpmem; 2816 break; 2817 default: 2818 BUG(); 2819 } 2820 2821 switch (MEMFILE_ATTR(cft->private)) { 2822 case RES_USAGE: 2823 if (counter == &memcg->memory) 2824 return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE; 2825 if (counter == &memcg->memsw) 2826 return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE; 2827 return (u64)page_counter_read(counter) * PAGE_SIZE; 2828 case RES_LIMIT: 2829 return (u64)counter->limit * PAGE_SIZE; 2830 case RES_MAX_USAGE: 2831 return (u64)counter->watermark * PAGE_SIZE; 2832 case RES_FAILCNT: 2833 return counter->failcnt; 2834 case RES_SOFT_LIMIT: 2835 return (u64)memcg->soft_limit * PAGE_SIZE; 2836 default: 2837 BUG(); 2838 } 2839 } 2840 2841 #ifndef CONFIG_SLOB 2842 static int memcg_online_kmem(struct mem_cgroup *memcg) 2843 { 2844 int memcg_id; 2845 2846 if (cgroup_memory_nokmem) 2847 return 0; 2848 2849 BUG_ON(memcg->kmemcg_id >= 0); 2850 BUG_ON(memcg->kmem_state); 2851 2852 memcg_id = memcg_alloc_cache_id(); 2853 if (memcg_id < 0) 2854 return memcg_id; 2855 2856 static_branch_inc(&memcg_kmem_enabled_key); 2857 /* 2858 * A memory cgroup is considered kmem-online as soon as it gets 2859 * kmemcg_id. Setting the id after enabling static branching will 2860 * guarantee no one starts accounting before all call sites are 2861 * patched. 2862 */ 2863 memcg->kmemcg_id = memcg_id; 2864 memcg->kmem_state = KMEM_ONLINE; 2865 INIT_LIST_HEAD(&memcg->kmem_caches); 2866 2867 return 0; 2868 } 2869 2870 static void memcg_offline_kmem(struct mem_cgroup *memcg) 2871 { 2872 struct cgroup_subsys_state *css; 2873 struct mem_cgroup *parent, *child; 2874 int kmemcg_id; 2875 2876 if (memcg->kmem_state != KMEM_ONLINE) 2877 return; 2878 /* 2879 * Clear the online state before clearing memcg_caches array 2880 * entries. The slab_mutex in memcg_deactivate_kmem_caches() 2881 * guarantees that no cache will be created for this cgroup 2882 * after we are done (see memcg_create_kmem_cache()). 2883 */ 2884 memcg->kmem_state = KMEM_ALLOCATED; 2885 2886 memcg_deactivate_kmem_caches(memcg); 2887 2888 kmemcg_id = memcg->kmemcg_id; 2889 BUG_ON(kmemcg_id < 0); 2890 2891 parent = parent_mem_cgroup(memcg); 2892 if (!parent) 2893 parent = root_mem_cgroup; 2894 2895 /* 2896 * Change kmemcg_id of this cgroup and all its descendants to the 2897 * parent's id, and then move all entries from this cgroup's list_lrus 2898 * to ones of the parent. After we have finished, all list_lrus 2899 * corresponding to this cgroup are guaranteed to remain empty. The 2900 * ordering is imposed by list_lru_node->lock taken by 2901 * memcg_drain_all_list_lrus(). 2902 */ 2903 rcu_read_lock(); /* can be called from css_free w/o cgroup_mutex */ 2904 css_for_each_descendant_pre(css, &memcg->css) { 2905 child = mem_cgroup_from_css(css); 2906 BUG_ON(child->kmemcg_id != kmemcg_id); 2907 child->kmemcg_id = parent->kmemcg_id; 2908 if (!memcg->use_hierarchy) 2909 break; 2910 } 2911 rcu_read_unlock(); 2912 2913 memcg_drain_all_list_lrus(kmemcg_id, parent->kmemcg_id); 2914 2915 memcg_free_cache_id(kmemcg_id); 2916 } 2917 2918 static void memcg_free_kmem(struct mem_cgroup *memcg) 2919 { 2920 /* css_alloc() failed, offlining didn't happen */ 2921 if (unlikely(memcg->kmem_state == KMEM_ONLINE)) 2922 memcg_offline_kmem(memcg); 2923 2924 if (memcg->kmem_state == KMEM_ALLOCATED) { 2925 memcg_destroy_kmem_caches(memcg); 2926 static_branch_dec(&memcg_kmem_enabled_key); 2927 WARN_ON(page_counter_read(&memcg->kmem)); 2928 } 2929 } 2930 #else 2931 static int memcg_online_kmem(struct mem_cgroup *memcg) 2932 { 2933 return 0; 2934 } 2935 static void memcg_offline_kmem(struct mem_cgroup *memcg) 2936 { 2937 } 2938 static void memcg_free_kmem(struct mem_cgroup *memcg) 2939 { 2940 } 2941 #endif /* !CONFIG_SLOB */ 2942 2943 static int memcg_update_kmem_limit(struct mem_cgroup *memcg, 2944 unsigned long limit) 2945 { 2946 int ret; 2947 2948 mutex_lock(&memcg_limit_mutex); 2949 ret = page_counter_limit(&memcg->kmem, limit); 2950 mutex_unlock(&memcg_limit_mutex); 2951 return ret; 2952 } 2953 2954 static int memcg_update_tcp_limit(struct mem_cgroup *memcg, unsigned long limit) 2955 { 2956 int ret; 2957 2958 mutex_lock(&memcg_limit_mutex); 2959 2960 ret = page_counter_limit(&memcg->tcpmem, limit); 2961 if (ret) 2962 goto out; 2963 2964 if (!memcg->tcpmem_active) { 2965 /* 2966 * The active flag needs to be written after the static_key 2967 * update. This is what guarantees that the socket activation 2968 * function is the last one to run. See mem_cgroup_sk_alloc() 2969 * for details, and note that we don't mark any socket as 2970 * belonging to this memcg until that flag is up. 2971 * 2972 * We need to do this, because static_keys will span multiple 2973 * sites, but we can't control their order. If we mark a socket 2974 * as accounted, but the accounting functions are not patched in 2975 * yet, we'll lose accounting. 2976 * 2977 * We never race with the readers in mem_cgroup_sk_alloc(), 2978 * because when this value change, the code to process it is not 2979 * patched in yet. 2980 */ 2981 static_branch_inc(&memcg_sockets_enabled_key); 2982 memcg->tcpmem_active = true; 2983 } 2984 out: 2985 mutex_unlock(&memcg_limit_mutex); 2986 return ret; 2987 } 2988 2989 /* 2990 * The user of this function is... 2991 * RES_LIMIT. 2992 */ 2993 static ssize_t mem_cgroup_write(struct kernfs_open_file *of, 2994 char *buf, size_t nbytes, loff_t off) 2995 { 2996 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 2997 unsigned long nr_pages; 2998 int ret; 2999 3000 buf = strstrip(buf); 3001 ret = page_counter_memparse(buf, "-1", &nr_pages); 3002 if (ret) 3003 return ret; 3004 3005 switch (MEMFILE_ATTR(of_cft(of)->private)) { 3006 case RES_LIMIT: 3007 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */ 3008 ret = -EINVAL; 3009 break; 3010 } 3011 switch (MEMFILE_TYPE(of_cft(of)->private)) { 3012 case _MEM: 3013 ret = mem_cgroup_resize_limit(memcg, nr_pages); 3014 break; 3015 case _MEMSWAP: 3016 ret = mem_cgroup_resize_memsw_limit(memcg, nr_pages); 3017 break; 3018 case _KMEM: 3019 ret = memcg_update_kmem_limit(memcg, nr_pages); 3020 break; 3021 case _TCP: 3022 ret = memcg_update_tcp_limit(memcg, nr_pages); 3023 break; 3024 } 3025 break; 3026 case RES_SOFT_LIMIT: 3027 memcg->soft_limit = nr_pages; 3028 ret = 0; 3029 break; 3030 } 3031 return ret ?: nbytes; 3032 } 3033 3034 static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf, 3035 size_t nbytes, loff_t off) 3036 { 3037 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 3038 struct page_counter *counter; 3039 3040 switch (MEMFILE_TYPE(of_cft(of)->private)) { 3041 case _MEM: 3042 counter = &memcg->memory; 3043 break; 3044 case _MEMSWAP: 3045 counter = &memcg->memsw; 3046 break; 3047 case _KMEM: 3048 counter = &memcg->kmem; 3049 break; 3050 case _TCP: 3051 counter = &memcg->tcpmem; 3052 break; 3053 default: 3054 BUG(); 3055 } 3056 3057 switch (MEMFILE_ATTR(of_cft(of)->private)) { 3058 case RES_MAX_USAGE: 3059 page_counter_reset_watermark(counter); 3060 break; 3061 case RES_FAILCNT: 3062 counter->failcnt = 0; 3063 break; 3064 default: 3065 BUG(); 3066 } 3067 3068 return nbytes; 3069 } 3070 3071 static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css, 3072 struct cftype *cft) 3073 { 3074 return mem_cgroup_from_css(css)->move_charge_at_immigrate; 3075 } 3076 3077 #ifdef CONFIG_MMU 3078 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css, 3079 struct cftype *cft, u64 val) 3080 { 3081 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 3082 3083 if (val & ~MOVE_MASK) 3084 return -EINVAL; 3085 3086 /* 3087 * No kind of locking is needed in here, because ->can_attach() will 3088 * check this value once in the beginning of the process, and then carry 3089 * on with stale data. This means that changes to this value will only 3090 * affect task migrations starting after the change. 3091 */ 3092 memcg->move_charge_at_immigrate = val; 3093 return 0; 3094 } 3095 #else 3096 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css, 3097 struct cftype *cft, u64 val) 3098 { 3099 return -ENOSYS; 3100 } 3101 #endif 3102 3103 #ifdef CONFIG_NUMA 3104 static int memcg_numa_stat_show(struct seq_file *m, void *v) 3105 { 3106 struct numa_stat { 3107 const char *name; 3108 unsigned int lru_mask; 3109 }; 3110 3111 static const struct numa_stat stats[] = { 3112 { "total", LRU_ALL }, 3113 { "file", LRU_ALL_FILE }, 3114 { "anon", LRU_ALL_ANON }, 3115 { "unevictable", BIT(LRU_UNEVICTABLE) }, 3116 }; 3117 const struct numa_stat *stat; 3118 int nid; 3119 unsigned long nr; 3120 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 3121 3122 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) { 3123 nr = mem_cgroup_nr_lru_pages(memcg, stat->lru_mask); 3124 seq_printf(m, "%s=%lu", stat->name, nr); 3125 for_each_node_state(nid, N_MEMORY) { 3126 nr = mem_cgroup_node_nr_lru_pages(memcg, nid, 3127 stat->lru_mask); 3128 seq_printf(m, " N%d=%lu", nid, nr); 3129 } 3130 seq_putc(m, '\n'); 3131 } 3132 3133 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) { 3134 struct mem_cgroup *iter; 3135 3136 nr = 0; 3137 for_each_mem_cgroup_tree(iter, memcg) 3138 nr += mem_cgroup_nr_lru_pages(iter, stat->lru_mask); 3139 seq_printf(m, "hierarchical_%s=%lu", stat->name, nr); 3140 for_each_node_state(nid, N_MEMORY) { 3141 nr = 0; 3142 for_each_mem_cgroup_tree(iter, memcg) 3143 nr += mem_cgroup_node_nr_lru_pages( 3144 iter, nid, stat->lru_mask); 3145 seq_printf(m, " N%d=%lu", nid, nr); 3146 } 3147 seq_putc(m, '\n'); 3148 } 3149 3150 return 0; 3151 } 3152 #endif /* CONFIG_NUMA */ 3153 3154 /* Universal VM events cgroup1 shows, original sort order */ 3155 unsigned int memcg1_events[] = { 3156 PGPGIN, 3157 PGPGOUT, 3158 PGFAULT, 3159 PGMAJFAULT, 3160 }; 3161 3162 static const char *const memcg1_event_names[] = { 3163 "pgpgin", 3164 "pgpgout", 3165 "pgfault", 3166 "pgmajfault", 3167 }; 3168 3169 static int memcg_stat_show(struct seq_file *m, void *v) 3170 { 3171 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 3172 unsigned long memory, memsw; 3173 struct mem_cgroup *mi; 3174 unsigned int i; 3175 3176 BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats)); 3177 BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS); 3178 3179 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) { 3180 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account()) 3181 continue; 3182 seq_printf(m, "%s %lu\n", memcg1_stat_names[i], 3183 memcg_page_state(memcg, memcg1_stats[i]) * 3184 PAGE_SIZE); 3185 } 3186 3187 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++) 3188 seq_printf(m, "%s %lu\n", memcg1_event_names[i], 3189 memcg_sum_events(memcg, memcg1_events[i])); 3190 3191 for (i = 0; i < NR_LRU_LISTS; i++) 3192 seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i], 3193 mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE); 3194 3195 /* Hierarchical information */ 3196 memory = memsw = PAGE_COUNTER_MAX; 3197 for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) { 3198 memory = min(memory, mi->memory.limit); 3199 memsw = min(memsw, mi->memsw.limit); 3200 } 3201 seq_printf(m, "hierarchical_memory_limit %llu\n", 3202 (u64)memory * PAGE_SIZE); 3203 if (do_memsw_account()) 3204 seq_printf(m, "hierarchical_memsw_limit %llu\n", 3205 (u64)memsw * PAGE_SIZE); 3206 3207 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) { 3208 unsigned long long val = 0; 3209 3210 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account()) 3211 continue; 3212 for_each_mem_cgroup_tree(mi, memcg) 3213 val += memcg_page_state(mi, memcg1_stats[i]) * 3214 PAGE_SIZE; 3215 seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i], val); 3216 } 3217 3218 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++) { 3219 unsigned long long val = 0; 3220 3221 for_each_mem_cgroup_tree(mi, memcg) 3222 val += memcg_sum_events(mi, memcg1_events[i]); 3223 seq_printf(m, "total_%s %llu\n", memcg1_event_names[i], val); 3224 } 3225 3226 for (i = 0; i < NR_LRU_LISTS; i++) { 3227 unsigned long long val = 0; 3228 3229 for_each_mem_cgroup_tree(mi, memcg) 3230 val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE; 3231 seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val); 3232 } 3233 3234 #ifdef CONFIG_DEBUG_VM 3235 { 3236 pg_data_t *pgdat; 3237 struct mem_cgroup_per_node *mz; 3238 struct zone_reclaim_stat *rstat; 3239 unsigned long recent_rotated[2] = {0, 0}; 3240 unsigned long recent_scanned[2] = {0, 0}; 3241 3242 for_each_online_pgdat(pgdat) { 3243 mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id); 3244 rstat = &mz->lruvec.reclaim_stat; 3245 3246 recent_rotated[0] += rstat->recent_rotated[0]; 3247 recent_rotated[1] += rstat->recent_rotated[1]; 3248 recent_scanned[0] += rstat->recent_scanned[0]; 3249 recent_scanned[1] += rstat->recent_scanned[1]; 3250 } 3251 seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]); 3252 seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]); 3253 seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]); 3254 seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]); 3255 } 3256 #endif 3257 3258 return 0; 3259 } 3260 3261 static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css, 3262 struct cftype *cft) 3263 { 3264 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 3265 3266 return mem_cgroup_swappiness(memcg); 3267 } 3268 3269 static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css, 3270 struct cftype *cft, u64 val) 3271 { 3272 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 3273 3274 if (val > 100) 3275 return -EINVAL; 3276 3277 if (css->parent) 3278 memcg->swappiness = val; 3279 else 3280 vm_swappiness = val; 3281 3282 return 0; 3283 } 3284 3285 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap) 3286 { 3287 struct mem_cgroup_threshold_ary *t; 3288 unsigned long usage; 3289 int i; 3290 3291 rcu_read_lock(); 3292 if (!swap) 3293 t = rcu_dereference(memcg->thresholds.primary); 3294 else 3295 t = rcu_dereference(memcg->memsw_thresholds.primary); 3296 3297 if (!t) 3298 goto unlock; 3299 3300 usage = mem_cgroup_usage(memcg, swap); 3301 3302 /* 3303 * current_threshold points to threshold just below or equal to usage. 3304 * If it's not true, a threshold was crossed after last 3305 * call of __mem_cgroup_threshold(). 3306 */ 3307 i = t->current_threshold; 3308 3309 /* 3310 * Iterate backward over array of thresholds starting from 3311 * current_threshold and check if a threshold is crossed. 3312 * If none of thresholds below usage is crossed, we read 3313 * only one element of the array here. 3314 */ 3315 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--) 3316 eventfd_signal(t->entries[i].eventfd, 1); 3317 3318 /* i = current_threshold + 1 */ 3319 i++; 3320 3321 /* 3322 * Iterate forward over array of thresholds starting from 3323 * current_threshold+1 and check if a threshold is crossed. 3324 * If none of thresholds above usage is crossed, we read 3325 * only one element of the array here. 3326 */ 3327 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++) 3328 eventfd_signal(t->entries[i].eventfd, 1); 3329 3330 /* Update current_threshold */ 3331 t->current_threshold = i - 1; 3332 unlock: 3333 rcu_read_unlock(); 3334 } 3335 3336 static void mem_cgroup_threshold(struct mem_cgroup *memcg) 3337 { 3338 while (memcg) { 3339 __mem_cgroup_threshold(memcg, false); 3340 if (do_memsw_account()) 3341 __mem_cgroup_threshold(memcg, true); 3342 3343 memcg = parent_mem_cgroup(memcg); 3344 } 3345 } 3346 3347 static int compare_thresholds(const void *a, const void *b) 3348 { 3349 const struct mem_cgroup_threshold *_a = a; 3350 const struct mem_cgroup_threshold *_b = b; 3351 3352 if (_a->threshold > _b->threshold) 3353 return 1; 3354 3355 if (_a->threshold < _b->threshold) 3356 return -1; 3357 3358 return 0; 3359 } 3360 3361 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg) 3362 { 3363 struct mem_cgroup_eventfd_list *ev; 3364 3365 spin_lock(&memcg_oom_lock); 3366 3367 list_for_each_entry(ev, &memcg->oom_notify, list) 3368 eventfd_signal(ev->eventfd, 1); 3369 3370 spin_unlock(&memcg_oom_lock); 3371 return 0; 3372 } 3373 3374 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg) 3375 { 3376 struct mem_cgroup *iter; 3377 3378 for_each_mem_cgroup_tree(iter, memcg) 3379 mem_cgroup_oom_notify_cb(iter); 3380 } 3381 3382 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg, 3383 struct eventfd_ctx *eventfd, const char *args, enum res_type type) 3384 { 3385 struct mem_cgroup_thresholds *thresholds; 3386 struct mem_cgroup_threshold_ary *new; 3387 unsigned long threshold; 3388 unsigned long usage; 3389 int i, size, ret; 3390 3391 ret = page_counter_memparse(args, "-1", &threshold); 3392 if (ret) 3393 return ret; 3394 3395 mutex_lock(&memcg->thresholds_lock); 3396 3397 if (type == _MEM) { 3398 thresholds = &memcg->thresholds; 3399 usage = mem_cgroup_usage(memcg, false); 3400 } else if (type == _MEMSWAP) { 3401 thresholds = &memcg->memsw_thresholds; 3402 usage = mem_cgroup_usage(memcg, true); 3403 } else 3404 BUG(); 3405 3406 /* Check if a threshold crossed before adding a new one */ 3407 if (thresholds->primary) 3408 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 3409 3410 size = thresholds->primary ? thresholds->primary->size + 1 : 1; 3411 3412 /* Allocate memory for new array of thresholds */ 3413 new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold), 3414 GFP_KERNEL); 3415 if (!new) { 3416 ret = -ENOMEM; 3417 goto unlock; 3418 } 3419 new->size = size; 3420 3421 /* Copy thresholds (if any) to new array */ 3422 if (thresholds->primary) { 3423 memcpy(new->entries, thresholds->primary->entries, (size - 1) * 3424 sizeof(struct mem_cgroup_threshold)); 3425 } 3426 3427 /* Add new threshold */ 3428 new->entries[size - 1].eventfd = eventfd; 3429 new->entries[size - 1].threshold = threshold; 3430 3431 /* Sort thresholds. Registering of new threshold isn't time-critical */ 3432 sort(new->entries, size, sizeof(struct mem_cgroup_threshold), 3433 compare_thresholds, NULL); 3434 3435 /* Find current threshold */ 3436 new->current_threshold = -1; 3437 for (i = 0; i < size; i++) { 3438 if (new->entries[i].threshold <= usage) { 3439 /* 3440 * new->current_threshold will not be used until 3441 * rcu_assign_pointer(), so it's safe to increment 3442 * it here. 3443 */ 3444 ++new->current_threshold; 3445 } else 3446 break; 3447 } 3448 3449 /* Free old spare buffer and save old primary buffer as spare */ 3450 kfree(thresholds->spare); 3451 thresholds->spare = thresholds->primary; 3452 3453 rcu_assign_pointer(thresholds->primary, new); 3454 3455 /* To be sure that nobody uses thresholds */ 3456 synchronize_rcu(); 3457 3458 unlock: 3459 mutex_unlock(&memcg->thresholds_lock); 3460 3461 return ret; 3462 } 3463 3464 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg, 3465 struct eventfd_ctx *eventfd, const char *args) 3466 { 3467 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM); 3468 } 3469 3470 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg, 3471 struct eventfd_ctx *eventfd, const char *args) 3472 { 3473 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP); 3474 } 3475 3476 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, 3477 struct eventfd_ctx *eventfd, enum res_type type) 3478 { 3479 struct mem_cgroup_thresholds *thresholds; 3480 struct mem_cgroup_threshold_ary *new; 3481 unsigned long usage; 3482 int i, j, size; 3483 3484 mutex_lock(&memcg->thresholds_lock); 3485 3486 if (type == _MEM) { 3487 thresholds = &memcg->thresholds; 3488 usage = mem_cgroup_usage(memcg, false); 3489 } else if (type == _MEMSWAP) { 3490 thresholds = &memcg->memsw_thresholds; 3491 usage = mem_cgroup_usage(memcg, true); 3492 } else 3493 BUG(); 3494 3495 if (!thresholds->primary) 3496 goto unlock; 3497 3498 /* Check if a threshold crossed before removing */ 3499 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 3500 3501 /* Calculate new number of threshold */ 3502 size = 0; 3503 for (i = 0; i < thresholds->primary->size; i++) { 3504 if (thresholds->primary->entries[i].eventfd != eventfd) 3505 size++; 3506 } 3507 3508 new = thresholds->spare; 3509 3510 /* Set thresholds array to NULL if we don't have thresholds */ 3511 if (!size) { 3512 kfree(new); 3513 new = NULL; 3514 goto swap_buffers; 3515 } 3516 3517 new->size = size; 3518 3519 /* Copy thresholds and find current threshold */ 3520 new->current_threshold = -1; 3521 for (i = 0, j = 0; i < thresholds->primary->size; i++) { 3522 if (thresholds->primary->entries[i].eventfd == eventfd) 3523 continue; 3524 3525 new->entries[j] = thresholds->primary->entries[i]; 3526 if (new->entries[j].threshold <= usage) { 3527 /* 3528 * new->current_threshold will not be used 3529 * until rcu_assign_pointer(), so it's safe to increment 3530 * it here. 3531 */ 3532 ++new->current_threshold; 3533 } 3534 j++; 3535 } 3536 3537 swap_buffers: 3538 /* Swap primary and spare array */ 3539 thresholds->spare = thresholds->primary; 3540 3541 rcu_assign_pointer(thresholds->primary, new); 3542 3543 /* To be sure that nobody uses thresholds */ 3544 synchronize_rcu(); 3545 3546 /* If all events are unregistered, free the spare array */ 3547 if (!new) { 3548 kfree(thresholds->spare); 3549 thresholds->spare = NULL; 3550 } 3551 unlock: 3552 mutex_unlock(&memcg->thresholds_lock); 3553 } 3554 3555 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, 3556 struct eventfd_ctx *eventfd) 3557 { 3558 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM); 3559 } 3560 3561 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg, 3562 struct eventfd_ctx *eventfd) 3563 { 3564 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP); 3565 } 3566 3567 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg, 3568 struct eventfd_ctx *eventfd, const char *args) 3569 { 3570 struct mem_cgroup_eventfd_list *event; 3571 3572 event = kmalloc(sizeof(*event), GFP_KERNEL); 3573 if (!event) 3574 return -ENOMEM; 3575 3576 spin_lock(&memcg_oom_lock); 3577 3578 event->eventfd = eventfd; 3579 list_add(&event->list, &memcg->oom_notify); 3580 3581 /* already in OOM ? */ 3582 if (memcg->under_oom) 3583 eventfd_signal(eventfd, 1); 3584 spin_unlock(&memcg_oom_lock); 3585 3586 return 0; 3587 } 3588 3589 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg, 3590 struct eventfd_ctx *eventfd) 3591 { 3592 struct mem_cgroup_eventfd_list *ev, *tmp; 3593 3594 spin_lock(&memcg_oom_lock); 3595 3596 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) { 3597 if (ev->eventfd == eventfd) { 3598 list_del(&ev->list); 3599 kfree(ev); 3600 } 3601 } 3602 3603 spin_unlock(&memcg_oom_lock); 3604 } 3605 3606 static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v) 3607 { 3608 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf)); 3609 3610 seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable); 3611 seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom); 3612 seq_printf(sf, "oom_kill %lu\n", memcg_sum_events(memcg, OOM_KILL)); 3613 return 0; 3614 } 3615 3616 static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css, 3617 struct cftype *cft, u64 val) 3618 { 3619 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 3620 3621 /* cannot set to root cgroup and only 0 and 1 are allowed */ 3622 if (!css->parent || !((val == 0) || (val == 1))) 3623 return -EINVAL; 3624 3625 memcg->oom_kill_disable = val; 3626 if (!val) 3627 memcg_oom_recover(memcg); 3628 3629 return 0; 3630 } 3631 3632 #ifdef CONFIG_CGROUP_WRITEBACK 3633 3634 struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg) 3635 { 3636 return &memcg->cgwb_list; 3637 } 3638 3639 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp) 3640 { 3641 return wb_domain_init(&memcg->cgwb_domain, gfp); 3642 } 3643 3644 static void memcg_wb_domain_exit(struct mem_cgroup *memcg) 3645 { 3646 wb_domain_exit(&memcg->cgwb_domain); 3647 } 3648 3649 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg) 3650 { 3651 wb_domain_size_changed(&memcg->cgwb_domain); 3652 } 3653 3654 struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb) 3655 { 3656 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); 3657 3658 if (!memcg->css.parent) 3659 return NULL; 3660 3661 return &memcg->cgwb_domain; 3662 } 3663 3664 /** 3665 * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg 3666 * @wb: bdi_writeback in question 3667 * @pfilepages: out parameter for number of file pages 3668 * @pheadroom: out parameter for number of allocatable pages according to memcg 3669 * @pdirty: out parameter for number of dirty pages 3670 * @pwriteback: out parameter for number of pages under writeback 3671 * 3672 * Determine the numbers of file, headroom, dirty, and writeback pages in 3673 * @wb's memcg. File, dirty and writeback are self-explanatory. Headroom 3674 * is a bit more involved. 3675 * 3676 * A memcg's headroom is "min(max, high) - used". In the hierarchy, the 3677 * headroom is calculated as the lowest headroom of itself and the 3678 * ancestors. Note that this doesn't consider the actual amount of 3679 * available memory in the system. The caller should further cap 3680 * *@pheadroom accordingly. 3681 */ 3682 void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages, 3683 unsigned long *pheadroom, unsigned long *pdirty, 3684 unsigned long *pwriteback) 3685 { 3686 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); 3687 struct mem_cgroup *parent; 3688 3689 *pdirty = memcg_page_state(memcg, NR_FILE_DIRTY); 3690 3691 /* this should eventually include NR_UNSTABLE_NFS */ 3692 *pwriteback = memcg_page_state(memcg, NR_WRITEBACK); 3693 *pfilepages = mem_cgroup_nr_lru_pages(memcg, (1 << LRU_INACTIVE_FILE) | 3694 (1 << LRU_ACTIVE_FILE)); 3695 *pheadroom = PAGE_COUNTER_MAX; 3696 3697 while ((parent = parent_mem_cgroup(memcg))) { 3698 unsigned long ceiling = min(memcg->memory.limit, memcg->high); 3699 unsigned long used = page_counter_read(&memcg->memory); 3700 3701 *pheadroom = min(*pheadroom, ceiling - min(ceiling, used)); 3702 memcg = parent; 3703 } 3704 } 3705 3706 #else /* CONFIG_CGROUP_WRITEBACK */ 3707 3708 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp) 3709 { 3710 return 0; 3711 } 3712 3713 static void memcg_wb_domain_exit(struct mem_cgroup *memcg) 3714 { 3715 } 3716 3717 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg) 3718 { 3719 } 3720 3721 #endif /* CONFIG_CGROUP_WRITEBACK */ 3722 3723 /* 3724 * DO NOT USE IN NEW FILES. 3725 * 3726 * "cgroup.event_control" implementation. 3727 * 3728 * This is way over-engineered. It tries to support fully configurable 3729 * events for each user. Such level of flexibility is completely 3730 * unnecessary especially in the light of the planned unified hierarchy. 3731 * 3732 * Please deprecate this and replace with something simpler if at all 3733 * possible. 3734 */ 3735 3736 /* 3737 * Unregister event and free resources. 3738 * 3739 * Gets called from workqueue. 3740 */ 3741 static void memcg_event_remove(struct work_struct *work) 3742 { 3743 struct mem_cgroup_event *event = 3744 container_of(work, struct mem_cgroup_event, remove); 3745 struct mem_cgroup *memcg = event->memcg; 3746 3747 remove_wait_queue(event->wqh, &event->wait); 3748 3749 event->unregister_event(memcg, event->eventfd); 3750 3751 /* Notify userspace the event is going away. */ 3752 eventfd_signal(event->eventfd, 1); 3753 3754 eventfd_ctx_put(event->eventfd); 3755 kfree(event); 3756 css_put(&memcg->css); 3757 } 3758 3759 /* 3760 * Gets called on POLLHUP on eventfd when user closes it. 3761 * 3762 * Called with wqh->lock held and interrupts disabled. 3763 */ 3764 static int memcg_event_wake(wait_queue_entry_t *wait, unsigned mode, 3765 int sync, void *key) 3766 { 3767 struct mem_cgroup_event *event = 3768 container_of(wait, struct mem_cgroup_event, wait); 3769 struct mem_cgroup *memcg = event->memcg; 3770 unsigned long flags = (unsigned long)key; 3771 3772 if (flags & POLLHUP) { 3773 /* 3774 * If the event has been detached at cgroup removal, we 3775 * can simply return knowing the other side will cleanup 3776 * for us. 3777 * 3778 * We can't race against event freeing since the other 3779 * side will require wqh->lock via remove_wait_queue(), 3780 * which we hold. 3781 */ 3782 spin_lock(&memcg->event_list_lock); 3783 if (!list_empty(&event->list)) { 3784 list_del_init(&event->list); 3785 /* 3786 * We are in atomic context, but cgroup_event_remove() 3787 * may sleep, so we have to call it in workqueue. 3788 */ 3789 schedule_work(&event->remove); 3790 } 3791 spin_unlock(&memcg->event_list_lock); 3792 } 3793 3794 return 0; 3795 } 3796 3797 static void memcg_event_ptable_queue_proc(struct file *file, 3798 wait_queue_head_t *wqh, poll_table *pt) 3799 { 3800 struct mem_cgroup_event *event = 3801 container_of(pt, struct mem_cgroup_event, pt); 3802 3803 event->wqh = wqh; 3804 add_wait_queue(wqh, &event->wait); 3805 } 3806 3807 /* 3808 * DO NOT USE IN NEW FILES. 3809 * 3810 * Parse input and register new cgroup event handler. 3811 * 3812 * Input must be in format '<event_fd> <control_fd> <args>'. 3813 * Interpretation of args is defined by control file implementation. 3814 */ 3815 static ssize_t memcg_write_event_control(struct kernfs_open_file *of, 3816 char *buf, size_t nbytes, loff_t off) 3817 { 3818 struct cgroup_subsys_state *css = of_css(of); 3819 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 3820 struct mem_cgroup_event *event; 3821 struct cgroup_subsys_state *cfile_css; 3822 unsigned int efd, cfd; 3823 struct fd efile; 3824 struct fd cfile; 3825 const char *name; 3826 char *endp; 3827 int ret; 3828 3829 buf = strstrip(buf); 3830 3831 efd = simple_strtoul(buf, &endp, 10); 3832 if (*endp != ' ') 3833 return -EINVAL; 3834 buf = endp + 1; 3835 3836 cfd = simple_strtoul(buf, &endp, 10); 3837 if ((*endp != ' ') && (*endp != '\0')) 3838 return -EINVAL; 3839 buf = endp + 1; 3840 3841 event = kzalloc(sizeof(*event), GFP_KERNEL); 3842 if (!event) 3843 return -ENOMEM; 3844 3845 event->memcg = memcg; 3846 INIT_LIST_HEAD(&event->list); 3847 init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc); 3848 init_waitqueue_func_entry(&event->wait, memcg_event_wake); 3849 INIT_WORK(&event->remove, memcg_event_remove); 3850 3851 efile = fdget(efd); 3852 if (!efile.file) { 3853 ret = -EBADF; 3854 goto out_kfree; 3855 } 3856 3857 event->eventfd = eventfd_ctx_fileget(efile.file); 3858 if (IS_ERR(event->eventfd)) { 3859 ret = PTR_ERR(event->eventfd); 3860 goto out_put_efile; 3861 } 3862 3863 cfile = fdget(cfd); 3864 if (!cfile.file) { 3865 ret = -EBADF; 3866 goto out_put_eventfd; 3867 } 3868 3869 /* the process need read permission on control file */ 3870 /* AV: shouldn't we check that it's been opened for read instead? */ 3871 ret = inode_permission(file_inode(cfile.file), MAY_READ); 3872 if (ret < 0) 3873 goto out_put_cfile; 3874 3875 /* 3876 * Determine the event callbacks and set them in @event. This used 3877 * to be done via struct cftype but cgroup core no longer knows 3878 * about these events. The following is crude but the whole thing 3879 * is for compatibility anyway. 3880 * 3881 * DO NOT ADD NEW FILES. 3882 */ 3883 name = cfile.file->f_path.dentry->d_name.name; 3884 3885 if (!strcmp(name, "memory.usage_in_bytes")) { 3886 event->register_event = mem_cgroup_usage_register_event; 3887 event->unregister_event = mem_cgroup_usage_unregister_event; 3888 } else if (!strcmp(name, "memory.oom_control")) { 3889 event->register_event = mem_cgroup_oom_register_event; 3890 event->unregister_event = mem_cgroup_oom_unregister_event; 3891 } else if (!strcmp(name, "memory.pressure_level")) { 3892 event->register_event = vmpressure_register_event; 3893 event->unregister_event = vmpressure_unregister_event; 3894 } else if (!strcmp(name, "memory.memsw.usage_in_bytes")) { 3895 event->register_event = memsw_cgroup_usage_register_event; 3896 event->unregister_event = memsw_cgroup_usage_unregister_event; 3897 } else { 3898 ret = -EINVAL; 3899 goto out_put_cfile; 3900 } 3901 3902 /* 3903 * Verify @cfile should belong to @css. Also, remaining events are 3904 * automatically removed on cgroup destruction but the removal is 3905 * asynchronous, so take an extra ref on @css. 3906 */ 3907 cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent, 3908 &memory_cgrp_subsys); 3909 ret = -EINVAL; 3910 if (IS_ERR(cfile_css)) 3911 goto out_put_cfile; 3912 if (cfile_css != css) { 3913 css_put(cfile_css); 3914 goto out_put_cfile; 3915 } 3916 3917 ret = event->register_event(memcg, event->eventfd, buf); 3918 if (ret) 3919 goto out_put_css; 3920 3921 efile.file->f_op->poll(efile.file, &event->pt); 3922 3923 spin_lock(&memcg->event_list_lock); 3924 list_add(&event->list, &memcg->event_list); 3925 spin_unlock(&memcg->event_list_lock); 3926 3927 fdput(cfile); 3928 fdput(efile); 3929 3930 return nbytes; 3931 3932 out_put_css: 3933 css_put(css); 3934 out_put_cfile: 3935 fdput(cfile); 3936 out_put_eventfd: 3937 eventfd_ctx_put(event->eventfd); 3938 out_put_efile: 3939 fdput(efile); 3940 out_kfree: 3941 kfree(event); 3942 3943 return ret; 3944 } 3945 3946 static struct cftype mem_cgroup_legacy_files[] = { 3947 { 3948 .name = "usage_in_bytes", 3949 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE), 3950 .read_u64 = mem_cgroup_read_u64, 3951 }, 3952 { 3953 .name = "max_usage_in_bytes", 3954 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE), 3955 .write = mem_cgroup_reset, 3956 .read_u64 = mem_cgroup_read_u64, 3957 }, 3958 { 3959 .name = "limit_in_bytes", 3960 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT), 3961 .write = mem_cgroup_write, 3962 .read_u64 = mem_cgroup_read_u64, 3963 }, 3964 { 3965 .name = "soft_limit_in_bytes", 3966 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT), 3967 .write = mem_cgroup_write, 3968 .read_u64 = mem_cgroup_read_u64, 3969 }, 3970 { 3971 .name = "failcnt", 3972 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT), 3973 .write = mem_cgroup_reset, 3974 .read_u64 = mem_cgroup_read_u64, 3975 }, 3976 { 3977 .name = "stat", 3978 .seq_show = memcg_stat_show, 3979 }, 3980 { 3981 .name = "force_empty", 3982 .write = mem_cgroup_force_empty_write, 3983 }, 3984 { 3985 .name = "use_hierarchy", 3986 .write_u64 = mem_cgroup_hierarchy_write, 3987 .read_u64 = mem_cgroup_hierarchy_read, 3988 }, 3989 { 3990 .name = "cgroup.event_control", /* XXX: for compat */ 3991 .write = memcg_write_event_control, 3992 .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE, 3993 }, 3994 { 3995 .name = "swappiness", 3996 .read_u64 = mem_cgroup_swappiness_read, 3997 .write_u64 = mem_cgroup_swappiness_write, 3998 }, 3999 { 4000 .name = "move_charge_at_immigrate", 4001 .read_u64 = mem_cgroup_move_charge_read, 4002 .write_u64 = mem_cgroup_move_charge_write, 4003 }, 4004 { 4005 .name = "oom_control", 4006 .seq_show = mem_cgroup_oom_control_read, 4007 .write_u64 = mem_cgroup_oom_control_write, 4008 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL), 4009 }, 4010 { 4011 .name = "pressure_level", 4012 }, 4013 #ifdef CONFIG_NUMA 4014 { 4015 .name = "numa_stat", 4016 .seq_show = memcg_numa_stat_show, 4017 }, 4018 #endif 4019 { 4020 .name = "kmem.limit_in_bytes", 4021 .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT), 4022 .write = mem_cgroup_write, 4023 .read_u64 = mem_cgroup_read_u64, 4024 }, 4025 { 4026 .name = "kmem.usage_in_bytes", 4027 .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE), 4028 .read_u64 = mem_cgroup_read_u64, 4029 }, 4030 { 4031 .name = "kmem.failcnt", 4032 .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT), 4033 .write = mem_cgroup_reset, 4034 .read_u64 = mem_cgroup_read_u64, 4035 }, 4036 { 4037 .name = "kmem.max_usage_in_bytes", 4038 .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE), 4039 .write = mem_cgroup_reset, 4040 .read_u64 = mem_cgroup_read_u64, 4041 }, 4042 #ifdef CONFIG_SLABINFO 4043 { 4044 .name = "kmem.slabinfo", 4045 .seq_start = memcg_slab_start, 4046 .seq_next = memcg_slab_next, 4047 .seq_stop = memcg_slab_stop, 4048 .seq_show = memcg_slab_show, 4049 }, 4050 #endif 4051 { 4052 .name = "kmem.tcp.limit_in_bytes", 4053 .private = MEMFILE_PRIVATE(_TCP, RES_LIMIT), 4054 .write = mem_cgroup_write, 4055 .read_u64 = mem_cgroup_read_u64, 4056 }, 4057 { 4058 .name = "kmem.tcp.usage_in_bytes", 4059 .private = MEMFILE_PRIVATE(_TCP, RES_USAGE), 4060 .read_u64 = mem_cgroup_read_u64, 4061 }, 4062 { 4063 .name = "kmem.tcp.failcnt", 4064 .private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT), 4065 .write = mem_cgroup_reset, 4066 .read_u64 = mem_cgroup_read_u64, 4067 }, 4068 { 4069 .name = "kmem.tcp.max_usage_in_bytes", 4070 .private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE), 4071 .write = mem_cgroup_reset, 4072 .read_u64 = mem_cgroup_read_u64, 4073 }, 4074 { }, /* terminate */ 4075 }; 4076 4077 /* 4078 * Private memory cgroup IDR 4079 * 4080 * Swap-out records and page cache shadow entries need to store memcg 4081 * references in constrained space, so we maintain an ID space that is 4082 * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of 4083 * memory-controlled cgroups to 64k. 4084 * 4085 * However, there usually are many references to the oflline CSS after 4086 * the cgroup has been destroyed, such as page cache or reclaimable 4087 * slab objects, that don't need to hang on to the ID. We want to keep 4088 * those dead CSS from occupying IDs, or we might quickly exhaust the 4089 * relatively small ID space and prevent the creation of new cgroups 4090 * even when there are much fewer than 64k cgroups - possibly none. 4091 * 4092 * Maintain a private 16-bit ID space for memcg, and allow the ID to 4093 * be freed and recycled when it's no longer needed, which is usually 4094 * when the CSS is offlined. 4095 * 4096 * The only exception to that are records of swapped out tmpfs/shmem 4097 * pages that need to be attributed to live ancestors on swapin. But 4098 * those references are manageable from userspace. 4099 */ 4100 4101 static DEFINE_IDR(mem_cgroup_idr); 4102 4103 static void mem_cgroup_id_get_many(struct mem_cgroup *memcg, unsigned int n) 4104 { 4105 VM_BUG_ON(atomic_read(&memcg->id.ref) <= 0); 4106 atomic_add(n, &memcg->id.ref); 4107 } 4108 4109 static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n) 4110 { 4111 VM_BUG_ON(atomic_read(&memcg->id.ref) < n); 4112 if (atomic_sub_and_test(n, &memcg->id.ref)) { 4113 idr_remove(&mem_cgroup_idr, memcg->id.id); 4114 memcg->id.id = 0; 4115 4116 /* Memcg ID pins CSS */ 4117 css_put(&memcg->css); 4118 } 4119 } 4120 4121 static inline void mem_cgroup_id_get(struct mem_cgroup *memcg) 4122 { 4123 mem_cgroup_id_get_many(memcg, 1); 4124 } 4125 4126 static inline void mem_cgroup_id_put(struct mem_cgroup *memcg) 4127 { 4128 mem_cgroup_id_put_many(memcg, 1); 4129 } 4130 4131 /** 4132 * mem_cgroup_from_id - look up a memcg from a memcg id 4133 * @id: the memcg id to look up 4134 * 4135 * Caller must hold rcu_read_lock(). 4136 */ 4137 struct mem_cgroup *mem_cgroup_from_id(unsigned short id) 4138 { 4139 WARN_ON_ONCE(!rcu_read_lock_held()); 4140 return idr_find(&mem_cgroup_idr, id); 4141 } 4142 4143 static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) 4144 { 4145 struct mem_cgroup_per_node *pn; 4146 int tmp = node; 4147 /* 4148 * This routine is called against possible nodes. 4149 * But it's BUG to call kmalloc() against offline node. 4150 * 4151 * TODO: this routine can waste much memory for nodes which will 4152 * never be onlined. It's better to use memory hotplug callback 4153 * function. 4154 */ 4155 if (!node_state(node, N_NORMAL_MEMORY)) 4156 tmp = -1; 4157 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp); 4158 if (!pn) 4159 return 1; 4160 4161 pn->lruvec_stat = alloc_percpu(struct lruvec_stat); 4162 if (!pn->lruvec_stat) { 4163 kfree(pn); 4164 return 1; 4165 } 4166 4167 lruvec_init(&pn->lruvec); 4168 pn->usage_in_excess = 0; 4169 pn->on_tree = false; 4170 pn->memcg = memcg; 4171 4172 memcg->nodeinfo[node] = pn; 4173 return 0; 4174 } 4175 4176 static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) 4177 { 4178 struct mem_cgroup_per_node *pn = memcg->nodeinfo[node]; 4179 4180 free_percpu(pn->lruvec_stat); 4181 kfree(pn); 4182 } 4183 4184 static void __mem_cgroup_free(struct mem_cgroup *memcg) 4185 { 4186 int node; 4187 4188 for_each_node(node) 4189 free_mem_cgroup_per_node_info(memcg, node); 4190 free_percpu(memcg->stat); 4191 kfree(memcg); 4192 } 4193 4194 static void mem_cgroup_free(struct mem_cgroup *memcg) 4195 { 4196 memcg_wb_domain_exit(memcg); 4197 __mem_cgroup_free(memcg); 4198 } 4199 4200 static struct mem_cgroup *mem_cgroup_alloc(void) 4201 { 4202 struct mem_cgroup *memcg; 4203 size_t size; 4204 int node; 4205 4206 size = sizeof(struct mem_cgroup); 4207 size += nr_node_ids * sizeof(struct mem_cgroup_per_node *); 4208 4209 memcg = kzalloc(size, GFP_KERNEL); 4210 if (!memcg) 4211 return NULL; 4212 4213 memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL, 4214 1, MEM_CGROUP_ID_MAX, 4215 GFP_KERNEL); 4216 if (memcg->id.id < 0) 4217 goto fail; 4218 4219 memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu); 4220 if (!memcg->stat) 4221 goto fail; 4222 4223 for_each_node(node) 4224 if (alloc_mem_cgroup_per_node_info(memcg, node)) 4225 goto fail; 4226 4227 if (memcg_wb_domain_init(memcg, GFP_KERNEL)) 4228 goto fail; 4229 4230 INIT_WORK(&memcg->high_work, high_work_func); 4231 memcg->last_scanned_node = MAX_NUMNODES; 4232 INIT_LIST_HEAD(&memcg->oom_notify); 4233 mutex_init(&memcg->thresholds_lock); 4234 spin_lock_init(&memcg->move_lock); 4235 vmpressure_init(&memcg->vmpressure); 4236 INIT_LIST_HEAD(&memcg->event_list); 4237 spin_lock_init(&memcg->event_list_lock); 4238 memcg->socket_pressure = jiffies; 4239 #ifndef CONFIG_SLOB 4240 memcg->kmemcg_id = -1; 4241 #endif 4242 #ifdef CONFIG_CGROUP_WRITEBACK 4243 INIT_LIST_HEAD(&memcg->cgwb_list); 4244 #endif 4245 idr_replace(&mem_cgroup_idr, memcg, memcg->id.id); 4246 return memcg; 4247 fail: 4248 if (memcg->id.id > 0) 4249 idr_remove(&mem_cgroup_idr, memcg->id.id); 4250 __mem_cgroup_free(memcg); 4251 return NULL; 4252 } 4253 4254 static struct cgroup_subsys_state * __ref 4255 mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 4256 { 4257 struct mem_cgroup *parent = mem_cgroup_from_css(parent_css); 4258 struct mem_cgroup *memcg; 4259 long error = -ENOMEM; 4260 4261 memcg = mem_cgroup_alloc(); 4262 if (!memcg) 4263 return ERR_PTR(error); 4264 4265 memcg->high = PAGE_COUNTER_MAX; 4266 memcg->soft_limit = PAGE_COUNTER_MAX; 4267 if (parent) { 4268 memcg->swappiness = mem_cgroup_swappiness(parent); 4269 memcg->oom_kill_disable = parent->oom_kill_disable; 4270 } 4271 if (parent && parent->use_hierarchy) { 4272 memcg->use_hierarchy = true; 4273 page_counter_init(&memcg->memory, &parent->memory); 4274 page_counter_init(&memcg->swap, &parent->swap); 4275 page_counter_init(&memcg->memsw, &parent->memsw); 4276 page_counter_init(&memcg->kmem, &parent->kmem); 4277 page_counter_init(&memcg->tcpmem, &parent->tcpmem); 4278 } else { 4279 page_counter_init(&memcg->memory, NULL); 4280 page_counter_init(&memcg->swap, NULL); 4281 page_counter_init(&memcg->memsw, NULL); 4282 page_counter_init(&memcg->kmem, NULL); 4283 page_counter_init(&memcg->tcpmem, NULL); 4284 /* 4285 * Deeper hierachy with use_hierarchy == false doesn't make 4286 * much sense so let cgroup subsystem know about this 4287 * unfortunate state in our controller. 4288 */ 4289 if (parent != root_mem_cgroup) 4290 memory_cgrp_subsys.broken_hierarchy = true; 4291 } 4292 4293 /* The following stuff does not apply to the root */ 4294 if (!parent) { 4295 root_mem_cgroup = memcg; 4296 return &memcg->css; 4297 } 4298 4299 error = memcg_online_kmem(memcg); 4300 if (error) 4301 goto fail; 4302 4303 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket) 4304 static_branch_inc(&memcg_sockets_enabled_key); 4305 4306 return &memcg->css; 4307 fail: 4308 mem_cgroup_free(memcg); 4309 return ERR_PTR(-ENOMEM); 4310 } 4311 4312 static int mem_cgroup_css_online(struct cgroup_subsys_state *css) 4313 { 4314 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4315 4316 /* Online state pins memcg ID, memcg ID pins CSS */ 4317 atomic_set(&memcg->id.ref, 1); 4318 css_get(css); 4319 return 0; 4320 } 4321 4322 static void mem_cgroup_css_offline(struct cgroup_subsys_state *css) 4323 { 4324 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4325 struct mem_cgroup_event *event, *tmp; 4326 4327 /* 4328 * Unregister events and notify userspace. 4329 * Notify userspace about cgroup removing only after rmdir of cgroup 4330 * directory to avoid race between userspace and kernelspace. 4331 */ 4332 spin_lock(&memcg->event_list_lock); 4333 list_for_each_entry_safe(event, tmp, &memcg->event_list, list) { 4334 list_del_init(&event->list); 4335 schedule_work(&event->remove); 4336 } 4337 spin_unlock(&memcg->event_list_lock); 4338 4339 memcg->low = 0; 4340 4341 memcg_offline_kmem(memcg); 4342 wb_memcg_offline(memcg); 4343 4344 mem_cgroup_id_put(memcg); 4345 } 4346 4347 static void mem_cgroup_css_released(struct cgroup_subsys_state *css) 4348 { 4349 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4350 4351 invalidate_reclaim_iterators(memcg); 4352 } 4353 4354 static void mem_cgroup_css_free(struct cgroup_subsys_state *css) 4355 { 4356 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4357 4358 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket) 4359 static_branch_dec(&memcg_sockets_enabled_key); 4360 4361 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active) 4362 static_branch_dec(&memcg_sockets_enabled_key); 4363 4364 vmpressure_cleanup(&memcg->vmpressure); 4365 cancel_work_sync(&memcg->high_work); 4366 mem_cgroup_remove_from_trees(memcg); 4367 memcg_free_kmem(memcg); 4368 mem_cgroup_free(memcg); 4369 } 4370 4371 /** 4372 * mem_cgroup_css_reset - reset the states of a mem_cgroup 4373 * @css: the target css 4374 * 4375 * Reset the states of the mem_cgroup associated with @css. This is 4376 * invoked when the userland requests disabling on the default hierarchy 4377 * but the memcg is pinned through dependency. The memcg should stop 4378 * applying policies and should revert to the vanilla state as it may be 4379 * made visible again. 4380 * 4381 * The current implementation only resets the essential configurations. 4382 * This needs to be expanded to cover all the visible parts. 4383 */ 4384 static void mem_cgroup_css_reset(struct cgroup_subsys_state *css) 4385 { 4386 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4387 4388 page_counter_limit(&memcg->memory, PAGE_COUNTER_MAX); 4389 page_counter_limit(&memcg->swap, PAGE_COUNTER_MAX); 4390 page_counter_limit(&memcg->memsw, PAGE_COUNTER_MAX); 4391 page_counter_limit(&memcg->kmem, PAGE_COUNTER_MAX); 4392 page_counter_limit(&memcg->tcpmem, PAGE_COUNTER_MAX); 4393 memcg->low = 0; 4394 memcg->high = PAGE_COUNTER_MAX; 4395 memcg->soft_limit = PAGE_COUNTER_MAX; 4396 memcg_wb_domain_size_changed(memcg); 4397 } 4398 4399 #ifdef CONFIG_MMU 4400 /* Handlers for move charge at task migration. */ 4401 static int mem_cgroup_do_precharge(unsigned long count) 4402 { 4403 int ret; 4404 4405 /* Try a single bulk charge without reclaim first, kswapd may wake */ 4406 ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count); 4407 if (!ret) { 4408 mc.precharge += count; 4409 return ret; 4410 } 4411 4412 /* Try charges one by one with reclaim, but do not retry */ 4413 while (count--) { 4414 ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1); 4415 if (ret) 4416 return ret; 4417 mc.precharge++; 4418 cond_resched(); 4419 } 4420 return 0; 4421 } 4422 4423 union mc_target { 4424 struct page *page; 4425 swp_entry_t ent; 4426 }; 4427 4428 enum mc_target_type { 4429 MC_TARGET_NONE = 0, 4430 MC_TARGET_PAGE, 4431 MC_TARGET_SWAP, 4432 MC_TARGET_DEVICE, 4433 }; 4434 4435 static struct page *mc_handle_present_pte(struct vm_area_struct *vma, 4436 unsigned long addr, pte_t ptent) 4437 { 4438 struct page *page = _vm_normal_page(vma, addr, ptent, true); 4439 4440 if (!page || !page_mapped(page)) 4441 return NULL; 4442 if (PageAnon(page)) { 4443 if (!(mc.flags & MOVE_ANON)) 4444 return NULL; 4445 } else { 4446 if (!(mc.flags & MOVE_FILE)) 4447 return NULL; 4448 } 4449 if (!get_page_unless_zero(page)) 4450 return NULL; 4451 4452 return page; 4453 } 4454 4455 #if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE) 4456 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma, 4457 pte_t ptent, swp_entry_t *entry) 4458 { 4459 struct page *page = NULL; 4460 swp_entry_t ent = pte_to_swp_entry(ptent); 4461 4462 if (!(mc.flags & MOVE_ANON) || non_swap_entry(ent)) 4463 return NULL; 4464 4465 /* 4466 * Handle MEMORY_DEVICE_PRIVATE which are ZONE_DEVICE page belonging to 4467 * a device and because they are not accessible by CPU they are store 4468 * as special swap entry in the CPU page table. 4469 */ 4470 if (is_device_private_entry(ent)) { 4471 page = device_private_entry_to_page(ent); 4472 /* 4473 * MEMORY_DEVICE_PRIVATE means ZONE_DEVICE page and which have 4474 * a refcount of 1 when free (unlike normal page) 4475 */ 4476 if (!page_ref_add_unless(page, 1, 1)) 4477 return NULL; 4478 return page; 4479 } 4480 4481 /* 4482 * Because lookup_swap_cache() updates some statistics counter, 4483 * we call find_get_page() with swapper_space directly. 4484 */ 4485 page = find_get_page(swap_address_space(ent), swp_offset(ent)); 4486 if (do_memsw_account()) 4487 entry->val = ent.val; 4488 4489 return page; 4490 } 4491 #else 4492 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma, 4493 pte_t ptent, swp_entry_t *entry) 4494 { 4495 return NULL; 4496 } 4497 #endif 4498 4499 static struct page *mc_handle_file_pte(struct vm_area_struct *vma, 4500 unsigned long addr, pte_t ptent, swp_entry_t *entry) 4501 { 4502 struct page *page = NULL; 4503 struct address_space *mapping; 4504 pgoff_t pgoff; 4505 4506 if (!vma->vm_file) /* anonymous vma */ 4507 return NULL; 4508 if (!(mc.flags & MOVE_FILE)) 4509 return NULL; 4510 4511 mapping = vma->vm_file->f_mapping; 4512 pgoff = linear_page_index(vma, addr); 4513 4514 /* page is moved even if it's not RSS of this task(page-faulted). */ 4515 #ifdef CONFIG_SWAP 4516 /* shmem/tmpfs may report page out on swap: account for that too. */ 4517 if (shmem_mapping(mapping)) { 4518 page = find_get_entry(mapping, pgoff); 4519 if (radix_tree_exceptional_entry(page)) { 4520 swp_entry_t swp = radix_to_swp_entry(page); 4521 if (do_memsw_account()) 4522 *entry = swp; 4523 page = find_get_page(swap_address_space(swp), 4524 swp_offset(swp)); 4525 } 4526 } else 4527 page = find_get_page(mapping, pgoff); 4528 #else 4529 page = find_get_page(mapping, pgoff); 4530 #endif 4531 return page; 4532 } 4533 4534 /** 4535 * mem_cgroup_move_account - move account of the page 4536 * @page: the page 4537 * @compound: charge the page as compound or small page 4538 * @from: mem_cgroup which the page is moved from. 4539 * @to: mem_cgroup which the page is moved to. @from != @to. 4540 * 4541 * The caller must make sure the page is not on LRU (isolate_page() is useful.) 4542 * 4543 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge" 4544 * from old cgroup. 4545 */ 4546 static int mem_cgroup_move_account(struct page *page, 4547 bool compound, 4548 struct mem_cgroup *from, 4549 struct mem_cgroup *to) 4550 { 4551 unsigned long flags; 4552 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1; 4553 int ret; 4554 bool anon; 4555 4556 VM_BUG_ON(from == to); 4557 VM_BUG_ON_PAGE(PageLRU(page), page); 4558 VM_BUG_ON(compound && !PageTransHuge(page)); 4559 4560 /* 4561 * Prevent mem_cgroup_migrate() from looking at 4562 * page->mem_cgroup of its source page while we change it. 4563 */ 4564 ret = -EBUSY; 4565 if (!trylock_page(page)) 4566 goto out; 4567 4568 ret = -EINVAL; 4569 if (page->mem_cgroup != from) 4570 goto out_unlock; 4571 4572 anon = PageAnon(page); 4573 4574 spin_lock_irqsave(&from->move_lock, flags); 4575 4576 if (!anon && page_mapped(page)) { 4577 __this_cpu_sub(from->stat->count[NR_FILE_MAPPED], nr_pages); 4578 __this_cpu_add(to->stat->count[NR_FILE_MAPPED], nr_pages); 4579 } 4580 4581 /* 4582 * move_lock grabbed above and caller set from->moving_account, so 4583 * mod_memcg_page_state will serialize updates to PageDirty. 4584 * So mapping should be stable for dirty pages. 4585 */ 4586 if (!anon && PageDirty(page)) { 4587 struct address_space *mapping = page_mapping(page); 4588 4589 if (mapping_cap_account_dirty(mapping)) { 4590 __this_cpu_sub(from->stat->count[NR_FILE_DIRTY], 4591 nr_pages); 4592 __this_cpu_add(to->stat->count[NR_FILE_DIRTY], 4593 nr_pages); 4594 } 4595 } 4596 4597 if (PageWriteback(page)) { 4598 __this_cpu_sub(from->stat->count[NR_WRITEBACK], nr_pages); 4599 __this_cpu_add(to->stat->count[NR_WRITEBACK], nr_pages); 4600 } 4601 4602 /* 4603 * It is safe to change page->mem_cgroup here because the page 4604 * is referenced, charged, and isolated - we can't race with 4605 * uncharging, charging, migration, or LRU putback. 4606 */ 4607 4608 /* caller should have done css_get */ 4609 page->mem_cgroup = to; 4610 spin_unlock_irqrestore(&from->move_lock, flags); 4611 4612 ret = 0; 4613 4614 local_irq_disable(); 4615 mem_cgroup_charge_statistics(to, page, compound, nr_pages); 4616 memcg_check_events(to, page); 4617 mem_cgroup_charge_statistics(from, page, compound, -nr_pages); 4618 memcg_check_events(from, page); 4619 local_irq_enable(); 4620 out_unlock: 4621 unlock_page(page); 4622 out: 4623 return ret; 4624 } 4625 4626 /** 4627 * get_mctgt_type - get target type of moving charge 4628 * @vma: the vma the pte to be checked belongs 4629 * @addr: the address corresponding to the pte to be checked 4630 * @ptent: the pte to be checked 4631 * @target: the pointer the target page or swap ent will be stored(can be NULL) 4632 * 4633 * Returns 4634 * 0(MC_TARGET_NONE): if the pte is not a target for move charge. 4635 * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for 4636 * move charge. if @target is not NULL, the page is stored in target->page 4637 * with extra refcnt got(Callers should handle it). 4638 * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a 4639 * target for charge migration. if @target is not NULL, the entry is stored 4640 * in target->ent. 4641 * 3(MC_TARGET_DEVICE): like MC_TARGET_PAGE but page is MEMORY_DEVICE_PUBLIC 4642 * or MEMORY_DEVICE_PRIVATE (so ZONE_DEVICE page and thus not on the lru). 4643 * For now we such page is charge like a regular page would be as for all 4644 * intent and purposes it is just special memory taking the place of a 4645 * regular page. 4646 * 4647 * See Documentations/vm/hmm.txt and include/linux/hmm.h 4648 * 4649 * Called with pte lock held. 4650 */ 4651 4652 static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma, 4653 unsigned long addr, pte_t ptent, union mc_target *target) 4654 { 4655 struct page *page = NULL; 4656 enum mc_target_type ret = MC_TARGET_NONE; 4657 swp_entry_t ent = { .val = 0 }; 4658 4659 if (pte_present(ptent)) 4660 page = mc_handle_present_pte(vma, addr, ptent); 4661 else if (is_swap_pte(ptent)) 4662 page = mc_handle_swap_pte(vma, ptent, &ent); 4663 else if (pte_none(ptent)) 4664 page = mc_handle_file_pte(vma, addr, ptent, &ent); 4665 4666 if (!page && !ent.val) 4667 return ret; 4668 if (page) { 4669 /* 4670 * Do only loose check w/o serialization. 4671 * mem_cgroup_move_account() checks the page is valid or 4672 * not under LRU exclusion. 4673 */ 4674 if (page->mem_cgroup == mc.from) { 4675 ret = MC_TARGET_PAGE; 4676 if (is_device_private_page(page) || 4677 is_device_public_page(page)) 4678 ret = MC_TARGET_DEVICE; 4679 if (target) 4680 target->page = page; 4681 } 4682 if (!ret || !target) 4683 put_page(page); 4684 } 4685 /* 4686 * There is a swap entry and a page doesn't exist or isn't charged. 4687 * But we cannot move a tail-page in a THP. 4688 */ 4689 if (ent.val && !ret && (!page || !PageTransCompound(page)) && 4690 mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) { 4691 ret = MC_TARGET_SWAP; 4692 if (target) 4693 target->ent = ent; 4694 } 4695 return ret; 4696 } 4697 4698 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4699 /* 4700 * We don't consider PMD mapped swapping or file mapped pages because THP does 4701 * not support them for now. 4702 * Caller should make sure that pmd_trans_huge(pmd) is true. 4703 */ 4704 static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma, 4705 unsigned long addr, pmd_t pmd, union mc_target *target) 4706 { 4707 struct page *page = NULL; 4708 enum mc_target_type ret = MC_TARGET_NONE; 4709 4710 if (unlikely(is_swap_pmd(pmd))) { 4711 VM_BUG_ON(thp_migration_supported() && 4712 !is_pmd_migration_entry(pmd)); 4713 return ret; 4714 } 4715 page = pmd_page(pmd); 4716 VM_BUG_ON_PAGE(!page || !PageHead(page), page); 4717 if (!(mc.flags & MOVE_ANON)) 4718 return ret; 4719 if (page->mem_cgroup == mc.from) { 4720 ret = MC_TARGET_PAGE; 4721 if (target) { 4722 get_page(page); 4723 target->page = page; 4724 } 4725 } 4726 return ret; 4727 } 4728 #else 4729 static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma, 4730 unsigned long addr, pmd_t pmd, union mc_target *target) 4731 { 4732 return MC_TARGET_NONE; 4733 } 4734 #endif 4735 4736 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd, 4737 unsigned long addr, unsigned long end, 4738 struct mm_walk *walk) 4739 { 4740 struct vm_area_struct *vma = walk->vma; 4741 pte_t *pte; 4742 spinlock_t *ptl; 4743 4744 ptl = pmd_trans_huge_lock(pmd, vma); 4745 if (ptl) { 4746 /* 4747 * Note their can not be MC_TARGET_DEVICE for now as we do not 4748 * support transparent huge page with MEMORY_DEVICE_PUBLIC or 4749 * MEMORY_DEVICE_PRIVATE but this might change. 4750 */ 4751 if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE) 4752 mc.precharge += HPAGE_PMD_NR; 4753 spin_unlock(ptl); 4754 return 0; 4755 } 4756 4757 if (pmd_trans_unstable(pmd)) 4758 return 0; 4759 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 4760 for (; addr != end; pte++, addr += PAGE_SIZE) 4761 if (get_mctgt_type(vma, addr, *pte, NULL)) 4762 mc.precharge++; /* increment precharge temporarily */ 4763 pte_unmap_unlock(pte - 1, ptl); 4764 cond_resched(); 4765 4766 return 0; 4767 } 4768 4769 static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm) 4770 { 4771 unsigned long precharge; 4772 4773 struct mm_walk mem_cgroup_count_precharge_walk = { 4774 .pmd_entry = mem_cgroup_count_precharge_pte_range, 4775 .mm = mm, 4776 }; 4777 down_read(&mm->mmap_sem); 4778 walk_page_range(0, mm->highest_vm_end, 4779 &mem_cgroup_count_precharge_walk); 4780 up_read(&mm->mmap_sem); 4781 4782 precharge = mc.precharge; 4783 mc.precharge = 0; 4784 4785 return precharge; 4786 } 4787 4788 static int mem_cgroup_precharge_mc(struct mm_struct *mm) 4789 { 4790 unsigned long precharge = mem_cgroup_count_precharge(mm); 4791 4792 VM_BUG_ON(mc.moving_task); 4793 mc.moving_task = current; 4794 return mem_cgroup_do_precharge(precharge); 4795 } 4796 4797 /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */ 4798 static void __mem_cgroup_clear_mc(void) 4799 { 4800 struct mem_cgroup *from = mc.from; 4801 struct mem_cgroup *to = mc.to; 4802 4803 /* we must uncharge all the leftover precharges from mc.to */ 4804 if (mc.precharge) { 4805 cancel_charge(mc.to, mc.precharge); 4806 mc.precharge = 0; 4807 } 4808 /* 4809 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so 4810 * we must uncharge here. 4811 */ 4812 if (mc.moved_charge) { 4813 cancel_charge(mc.from, mc.moved_charge); 4814 mc.moved_charge = 0; 4815 } 4816 /* we must fixup refcnts and charges */ 4817 if (mc.moved_swap) { 4818 /* uncharge swap account from the old cgroup */ 4819 if (!mem_cgroup_is_root(mc.from)) 4820 page_counter_uncharge(&mc.from->memsw, mc.moved_swap); 4821 4822 mem_cgroup_id_put_many(mc.from, mc.moved_swap); 4823 4824 /* 4825 * we charged both to->memory and to->memsw, so we 4826 * should uncharge to->memory. 4827 */ 4828 if (!mem_cgroup_is_root(mc.to)) 4829 page_counter_uncharge(&mc.to->memory, mc.moved_swap); 4830 4831 mem_cgroup_id_get_many(mc.to, mc.moved_swap); 4832 css_put_many(&mc.to->css, mc.moved_swap); 4833 4834 mc.moved_swap = 0; 4835 } 4836 memcg_oom_recover(from); 4837 memcg_oom_recover(to); 4838 wake_up_all(&mc.waitq); 4839 } 4840 4841 static void mem_cgroup_clear_mc(void) 4842 { 4843 struct mm_struct *mm = mc.mm; 4844 4845 /* 4846 * we must clear moving_task before waking up waiters at the end of 4847 * task migration. 4848 */ 4849 mc.moving_task = NULL; 4850 __mem_cgroup_clear_mc(); 4851 spin_lock(&mc.lock); 4852 mc.from = NULL; 4853 mc.to = NULL; 4854 mc.mm = NULL; 4855 spin_unlock(&mc.lock); 4856 4857 mmput(mm); 4858 } 4859 4860 static int mem_cgroup_can_attach(struct cgroup_taskset *tset) 4861 { 4862 struct cgroup_subsys_state *css; 4863 struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */ 4864 struct mem_cgroup *from; 4865 struct task_struct *leader, *p; 4866 struct mm_struct *mm; 4867 unsigned long move_flags; 4868 int ret = 0; 4869 4870 /* charge immigration isn't supported on the default hierarchy */ 4871 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 4872 return 0; 4873 4874 /* 4875 * Multi-process migrations only happen on the default hierarchy 4876 * where charge immigration is not used. Perform charge 4877 * immigration if @tset contains a leader and whine if there are 4878 * multiple. 4879 */ 4880 p = NULL; 4881 cgroup_taskset_for_each_leader(leader, css, tset) { 4882 WARN_ON_ONCE(p); 4883 p = leader; 4884 memcg = mem_cgroup_from_css(css); 4885 } 4886 if (!p) 4887 return 0; 4888 4889 /* 4890 * We are now commited to this value whatever it is. Changes in this 4891 * tunable will only affect upcoming migrations, not the current one. 4892 * So we need to save it, and keep it going. 4893 */ 4894 move_flags = READ_ONCE(memcg->move_charge_at_immigrate); 4895 if (!move_flags) 4896 return 0; 4897 4898 from = mem_cgroup_from_task(p); 4899 4900 VM_BUG_ON(from == memcg); 4901 4902 mm = get_task_mm(p); 4903 if (!mm) 4904 return 0; 4905 /* We move charges only when we move a owner of the mm */ 4906 if (mm->owner == p) { 4907 VM_BUG_ON(mc.from); 4908 VM_BUG_ON(mc.to); 4909 VM_BUG_ON(mc.precharge); 4910 VM_BUG_ON(mc.moved_charge); 4911 VM_BUG_ON(mc.moved_swap); 4912 4913 spin_lock(&mc.lock); 4914 mc.mm = mm; 4915 mc.from = from; 4916 mc.to = memcg; 4917 mc.flags = move_flags; 4918 spin_unlock(&mc.lock); 4919 /* We set mc.moving_task later */ 4920 4921 ret = mem_cgroup_precharge_mc(mm); 4922 if (ret) 4923 mem_cgroup_clear_mc(); 4924 } else { 4925 mmput(mm); 4926 } 4927 return ret; 4928 } 4929 4930 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset) 4931 { 4932 if (mc.to) 4933 mem_cgroup_clear_mc(); 4934 } 4935 4936 static int mem_cgroup_move_charge_pte_range(pmd_t *pmd, 4937 unsigned long addr, unsigned long end, 4938 struct mm_walk *walk) 4939 { 4940 int ret = 0; 4941 struct vm_area_struct *vma = walk->vma; 4942 pte_t *pte; 4943 spinlock_t *ptl; 4944 enum mc_target_type target_type; 4945 union mc_target target; 4946 struct page *page; 4947 4948 ptl = pmd_trans_huge_lock(pmd, vma); 4949 if (ptl) { 4950 if (mc.precharge < HPAGE_PMD_NR) { 4951 spin_unlock(ptl); 4952 return 0; 4953 } 4954 target_type = get_mctgt_type_thp(vma, addr, *pmd, &target); 4955 if (target_type == MC_TARGET_PAGE) { 4956 page = target.page; 4957 if (!isolate_lru_page(page)) { 4958 if (!mem_cgroup_move_account(page, true, 4959 mc.from, mc.to)) { 4960 mc.precharge -= HPAGE_PMD_NR; 4961 mc.moved_charge += HPAGE_PMD_NR; 4962 } 4963 putback_lru_page(page); 4964 } 4965 put_page(page); 4966 } else if (target_type == MC_TARGET_DEVICE) { 4967 page = target.page; 4968 if (!mem_cgroup_move_account(page, true, 4969 mc.from, mc.to)) { 4970 mc.precharge -= HPAGE_PMD_NR; 4971 mc.moved_charge += HPAGE_PMD_NR; 4972 } 4973 put_page(page); 4974 } 4975 spin_unlock(ptl); 4976 return 0; 4977 } 4978 4979 if (pmd_trans_unstable(pmd)) 4980 return 0; 4981 retry: 4982 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 4983 for (; addr != end; addr += PAGE_SIZE) { 4984 pte_t ptent = *(pte++); 4985 bool device = false; 4986 swp_entry_t ent; 4987 4988 if (!mc.precharge) 4989 break; 4990 4991 switch (get_mctgt_type(vma, addr, ptent, &target)) { 4992 case MC_TARGET_DEVICE: 4993 device = true; 4994 /* fall through */ 4995 case MC_TARGET_PAGE: 4996 page = target.page; 4997 /* 4998 * We can have a part of the split pmd here. Moving it 4999 * can be done but it would be too convoluted so simply 5000 * ignore such a partial THP and keep it in original 5001 * memcg. There should be somebody mapping the head. 5002 */ 5003 if (PageTransCompound(page)) 5004 goto put; 5005 if (!device && isolate_lru_page(page)) 5006 goto put; 5007 if (!mem_cgroup_move_account(page, false, 5008 mc.from, mc.to)) { 5009 mc.precharge--; 5010 /* we uncharge from mc.from later. */ 5011 mc.moved_charge++; 5012 } 5013 if (!device) 5014 putback_lru_page(page); 5015 put: /* get_mctgt_type() gets the page */ 5016 put_page(page); 5017 break; 5018 case MC_TARGET_SWAP: 5019 ent = target.ent; 5020 if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) { 5021 mc.precharge--; 5022 /* we fixup refcnts and charges later. */ 5023 mc.moved_swap++; 5024 } 5025 break; 5026 default: 5027 break; 5028 } 5029 } 5030 pte_unmap_unlock(pte - 1, ptl); 5031 cond_resched(); 5032 5033 if (addr != end) { 5034 /* 5035 * We have consumed all precharges we got in can_attach(). 5036 * We try charge one by one, but don't do any additional 5037 * charges to mc.to if we have failed in charge once in attach() 5038 * phase. 5039 */ 5040 ret = mem_cgroup_do_precharge(1); 5041 if (!ret) 5042 goto retry; 5043 } 5044 5045 return ret; 5046 } 5047 5048 static void mem_cgroup_move_charge(void) 5049 { 5050 struct mm_walk mem_cgroup_move_charge_walk = { 5051 .pmd_entry = mem_cgroup_move_charge_pte_range, 5052 .mm = mc.mm, 5053 }; 5054 5055 lru_add_drain_all(); 5056 /* 5057 * Signal lock_page_memcg() to take the memcg's move_lock 5058 * while we're moving its pages to another memcg. Then wait 5059 * for already started RCU-only updates to finish. 5060 */ 5061 atomic_inc(&mc.from->moving_account); 5062 synchronize_rcu(); 5063 retry: 5064 if (unlikely(!down_read_trylock(&mc.mm->mmap_sem))) { 5065 /* 5066 * Someone who are holding the mmap_sem might be waiting in 5067 * waitq. So we cancel all extra charges, wake up all waiters, 5068 * and retry. Because we cancel precharges, we might not be able 5069 * to move enough charges, but moving charge is a best-effort 5070 * feature anyway, so it wouldn't be a big problem. 5071 */ 5072 __mem_cgroup_clear_mc(); 5073 cond_resched(); 5074 goto retry; 5075 } 5076 /* 5077 * When we have consumed all precharges and failed in doing 5078 * additional charge, the page walk just aborts. 5079 */ 5080 walk_page_range(0, mc.mm->highest_vm_end, &mem_cgroup_move_charge_walk); 5081 5082 up_read(&mc.mm->mmap_sem); 5083 atomic_dec(&mc.from->moving_account); 5084 } 5085 5086 static void mem_cgroup_move_task(void) 5087 { 5088 if (mc.to) { 5089 mem_cgroup_move_charge(); 5090 mem_cgroup_clear_mc(); 5091 } 5092 } 5093 #else /* !CONFIG_MMU */ 5094 static int mem_cgroup_can_attach(struct cgroup_taskset *tset) 5095 { 5096 return 0; 5097 } 5098 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset) 5099 { 5100 } 5101 static void mem_cgroup_move_task(void) 5102 { 5103 } 5104 #endif 5105 5106 /* 5107 * Cgroup retains root cgroups across [un]mount cycles making it necessary 5108 * to verify whether we're attached to the default hierarchy on each mount 5109 * attempt. 5110 */ 5111 static void mem_cgroup_bind(struct cgroup_subsys_state *root_css) 5112 { 5113 /* 5114 * use_hierarchy is forced on the default hierarchy. cgroup core 5115 * guarantees that @root doesn't have any children, so turning it 5116 * on for the root memcg is enough. 5117 */ 5118 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 5119 root_mem_cgroup->use_hierarchy = true; 5120 else 5121 root_mem_cgroup->use_hierarchy = false; 5122 } 5123 5124 static u64 memory_current_read(struct cgroup_subsys_state *css, 5125 struct cftype *cft) 5126 { 5127 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5128 5129 return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE; 5130 } 5131 5132 static int memory_low_show(struct seq_file *m, void *v) 5133 { 5134 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 5135 unsigned long low = READ_ONCE(memcg->low); 5136 5137 if (low == PAGE_COUNTER_MAX) 5138 seq_puts(m, "max\n"); 5139 else 5140 seq_printf(m, "%llu\n", (u64)low * PAGE_SIZE); 5141 5142 return 0; 5143 } 5144 5145 static ssize_t memory_low_write(struct kernfs_open_file *of, 5146 char *buf, size_t nbytes, loff_t off) 5147 { 5148 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 5149 unsigned long low; 5150 int err; 5151 5152 buf = strstrip(buf); 5153 err = page_counter_memparse(buf, "max", &low); 5154 if (err) 5155 return err; 5156 5157 memcg->low = low; 5158 5159 return nbytes; 5160 } 5161 5162 static int memory_high_show(struct seq_file *m, void *v) 5163 { 5164 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 5165 unsigned long high = READ_ONCE(memcg->high); 5166 5167 if (high == PAGE_COUNTER_MAX) 5168 seq_puts(m, "max\n"); 5169 else 5170 seq_printf(m, "%llu\n", (u64)high * PAGE_SIZE); 5171 5172 return 0; 5173 } 5174 5175 static ssize_t memory_high_write(struct kernfs_open_file *of, 5176 char *buf, size_t nbytes, loff_t off) 5177 { 5178 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 5179 unsigned long nr_pages; 5180 unsigned long high; 5181 int err; 5182 5183 buf = strstrip(buf); 5184 err = page_counter_memparse(buf, "max", &high); 5185 if (err) 5186 return err; 5187 5188 memcg->high = high; 5189 5190 nr_pages = page_counter_read(&memcg->memory); 5191 if (nr_pages > high) 5192 try_to_free_mem_cgroup_pages(memcg, nr_pages - high, 5193 GFP_KERNEL, true); 5194 5195 memcg_wb_domain_size_changed(memcg); 5196 return nbytes; 5197 } 5198 5199 static int memory_max_show(struct seq_file *m, void *v) 5200 { 5201 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 5202 unsigned long max = READ_ONCE(memcg->memory.limit); 5203 5204 if (max == PAGE_COUNTER_MAX) 5205 seq_puts(m, "max\n"); 5206 else 5207 seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE); 5208 5209 return 0; 5210 } 5211 5212 static ssize_t memory_max_write(struct kernfs_open_file *of, 5213 char *buf, size_t nbytes, loff_t off) 5214 { 5215 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 5216 unsigned int nr_reclaims = MEM_CGROUP_RECLAIM_RETRIES; 5217 bool drained = false; 5218 unsigned long max; 5219 int err; 5220 5221 buf = strstrip(buf); 5222 err = page_counter_memparse(buf, "max", &max); 5223 if (err) 5224 return err; 5225 5226 xchg(&memcg->memory.limit, max); 5227 5228 for (;;) { 5229 unsigned long nr_pages = page_counter_read(&memcg->memory); 5230 5231 if (nr_pages <= max) 5232 break; 5233 5234 if (signal_pending(current)) { 5235 err = -EINTR; 5236 break; 5237 } 5238 5239 if (!drained) { 5240 drain_all_stock(memcg); 5241 drained = true; 5242 continue; 5243 } 5244 5245 if (nr_reclaims) { 5246 if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max, 5247 GFP_KERNEL, true)) 5248 nr_reclaims--; 5249 continue; 5250 } 5251 5252 mem_cgroup_event(memcg, MEMCG_OOM); 5253 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0)) 5254 break; 5255 } 5256 5257 memcg_wb_domain_size_changed(memcg); 5258 return nbytes; 5259 } 5260 5261 static int memory_events_show(struct seq_file *m, void *v) 5262 { 5263 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 5264 5265 seq_printf(m, "low %lu\n", memcg_sum_events(memcg, MEMCG_LOW)); 5266 seq_printf(m, "high %lu\n", memcg_sum_events(memcg, MEMCG_HIGH)); 5267 seq_printf(m, "max %lu\n", memcg_sum_events(memcg, MEMCG_MAX)); 5268 seq_printf(m, "oom %lu\n", memcg_sum_events(memcg, MEMCG_OOM)); 5269 seq_printf(m, "oom_kill %lu\n", memcg_sum_events(memcg, OOM_KILL)); 5270 5271 return 0; 5272 } 5273 5274 static int memory_stat_show(struct seq_file *m, void *v) 5275 { 5276 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 5277 unsigned long stat[MEMCG_NR_STAT]; 5278 unsigned long events[MEMCG_NR_EVENTS]; 5279 int i; 5280 5281 /* 5282 * Provide statistics on the state of the memory subsystem as 5283 * well as cumulative event counters that show past behavior. 5284 * 5285 * This list is ordered following a combination of these gradients: 5286 * 1) generic big picture -> specifics and details 5287 * 2) reflecting userspace activity -> reflecting kernel heuristics 5288 * 5289 * Current memory state: 5290 */ 5291 5292 tree_stat(memcg, stat); 5293 tree_events(memcg, events); 5294 5295 seq_printf(m, "anon %llu\n", 5296 (u64)stat[MEMCG_RSS] * PAGE_SIZE); 5297 seq_printf(m, "file %llu\n", 5298 (u64)stat[MEMCG_CACHE] * PAGE_SIZE); 5299 seq_printf(m, "kernel_stack %llu\n", 5300 (u64)stat[MEMCG_KERNEL_STACK_KB] * 1024); 5301 seq_printf(m, "slab %llu\n", 5302 (u64)(stat[NR_SLAB_RECLAIMABLE] + 5303 stat[NR_SLAB_UNRECLAIMABLE]) * PAGE_SIZE); 5304 seq_printf(m, "sock %llu\n", 5305 (u64)stat[MEMCG_SOCK] * PAGE_SIZE); 5306 5307 seq_printf(m, "shmem %llu\n", 5308 (u64)stat[NR_SHMEM] * PAGE_SIZE); 5309 seq_printf(m, "file_mapped %llu\n", 5310 (u64)stat[NR_FILE_MAPPED] * PAGE_SIZE); 5311 seq_printf(m, "file_dirty %llu\n", 5312 (u64)stat[NR_FILE_DIRTY] * PAGE_SIZE); 5313 seq_printf(m, "file_writeback %llu\n", 5314 (u64)stat[NR_WRITEBACK] * PAGE_SIZE); 5315 5316 for (i = 0; i < NR_LRU_LISTS; i++) { 5317 struct mem_cgroup *mi; 5318 unsigned long val = 0; 5319 5320 for_each_mem_cgroup_tree(mi, memcg) 5321 val += mem_cgroup_nr_lru_pages(mi, BIT(i)); 5322 seq_printf(m, "%s %llu\n", 5323 mem_cgroup_lru_names[i], (u64)val * PAGE_SIZE); 5324 } 5325 5326 seq_printf(m, "slab_reclaimable %llu\n", 5327 (u64)stat[NR_SLAB_RECLAIMABLE] * PAGE_SIZE); 5328 seq_printf(m, "slab_unreclaimable %llu\n", 5329 (u64)stat[NR_SLAB_UNRECLAIMABLE] * PAGE_SIZE); 5330 5331 /* Accumulated memory events */ 5332 5333 seq_printf(m, "pgfault %lu\n", events[PGFAULT]); 5334 seq_printf(m, "pgmajfault %lu\n", events[PGMAJFAULT]); 5335 5336 seq_printf(m, "pgrefill %lu\n", events[PGREFILL]); 5337 seq_printf(m, "pgscan %lu\n", events[PGSCAN_KSWAPD] + 5338 events[PGSCAN_DIRECT]); 5339 seq_printf(m, "pgsteal %lu\n", events[PGSTEAL_KSWAPD] + 5340 events[PGSTEAL_DIRECT]); 5341 seq_printf(m, "pgactivate %lu\n", events[PGACTIVATE]); 5342 seq_printf(m, "pgdeactivate %lu\n", events[PGDEACTIVATE]); 5343 seq_printf(m, "pglazyfree %lu\n", events[PGLAZYFREE]); 5344 seq_printf(m, "pglazyfreed %lu\n", events[PGLAZYFREED]); 5345 5346 seq_printf(m, "workingset_refault %lu\n", 5347 stat[WORKINGSET_REFAULT]); 5348 seq_printf(m, "workingset_activate %lu\n", 5349 stat[WORKINGSET_ACTIVATE]); 5350 seq_printf(m, "workingset_nodereclaim %lu\n", 5351 stat[WORKINGSET_NODERECLAIM]); 5352 5353 return 0; 5354 } 5355 5356 static struct cftype memory_files[] = { 5357 { 5358 .name = "current", 5359 .flags = CFTYPE_NOT_ON_ROOT, 5360 .read_u64 = memory_current_read, 5361 }, 5362 { 5363 .name = "low", 5364 .flags = CFTYPE_NOT_ON_ROOT, 5365 .seq_show = memory_low_show, 5366 .write = memory_low_write, 5367 }, 5368 { 5369 .name = "high", 5370 .flags = CFTYPE_NOT_ON_ROOT, 5371 .seq_show = memory_high_show, 5372 .write = memory_high_write, 5373 }, 5374 { 5375 .name = "max", 5376 .flags = CFTYPE_NOT_ON_ROOT, 5377 .seq_show = memory_max_show, 5378 .write = memory_max_write, 5379 }, 5380 { 5381 .name = "events", 5382 .flags = CFTYPE_NOT_ON_ROOT, 5383 .file_offset = offsetof(struct mem_cgroup, events_file), 5384 .seq_show = memory_events_show, 5385 }, 5386 { 5387 .name = "stat", 5388 .flags = CFTYPE_NOT_ON_ROOT, 5389 .seq_show = memory_stat_show, 5390 }, 5391 { } /* terminate */ 5392 }; 5393 5394 struct cgroup_subsys memory_cgrp_subsys = { 5395 .css_alloc = mem_cgroup_css_alloc, 5396 .css_online = mem_cgroup_css_online, 5397 .css_offline = mem_cgroup_css_offline, 5398 .css_released = mem_cgroup_css_released, 5399 .css_free = mem_cgroup_css_free, 5400 .css_reset = mem_cgroup_css_reset, 5401 .can_attach = mem_cgroup_can_attach, 5402 .cancel_attach = mem_cgroup_cancel_attach, 5403 .post_attach = mem_cgroup_move_task, 5404 .bind = mem_cgroup_bind, 5405 .dfl_cftypes = memory_files, 5406 .legacy_cftypes = mem_cgroup_legacy_files, 5407 .early_init = 0, 5408 }; 5409 5410 /** 5411 * mem_cgroup_low - check if memory consumption is below the normal range 5412 * @root: the top ancestor of the sub-tree being checked 5413 * @memcg: the memory cgroup to check 5414 * 5415 * Returns %true if memory consumption of @memcg, and that of all 5416 * ancestors up to (but not including) @root, is below the normal range. 5417 * 5418 * @root is exclusive; it is never low when looked at directly and isn't 5419 * checked when traversing the hierarchy. 5420 * 5421 * Excluding @root enables using memory.low to prioritize memory usage 5422 * between cgroups within a subtree of the hierarchy that is limited by 5423 * memory.high or memory.max. 5424 * 5425 * For example, given cgroup A with children B and C: 5426 * 5427 * A 5428 * / \ 5429 * B C 5430 * 5431 * and 5432 * 5433 * 1. A/memory.current > A/memory.high 5434 * 2. A/B/memory.current < A/B/memory.low 5435 * 3. A/C/memory.current >= A/C/memory.low 5436 * 5437 * As 'A' is high, i.e. triggers reclaim from 'A', and 'B' is low, we 5438 * should reclaim from 'C' until 'A' is no longer high or until we can 5439 * no longer reclaim from 'C'. If 'A', i.e. @root, isn't excluded by 5440 * mem_cgroup_low when reclaming from 'A', then 'B' won't be considered 5441 * low and we will reclaim indiscriminately from both 'B' and 'C'. 5442 */ 5443 bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg) 5444 { 5445 if (mem_cgroup_disabled()) 5446 return false; 5447 5448 if (!root) 5449 root = root_mem_cgroup; 5450 if (memcg == root) 5451 return false; 5452 5453 for (; memcg != root; memcg = parent_mem_cgroup(memcg)) { 5454 if (page_counter_read(&memcg->memory) >= memcg->low) 5455 return false; 5456 } 5457 5458 return true; 5459 } 5460 5461 /** 5462 * mem_cgroup_try_charge - try charging a page 5463 * @page: page to charge 5464 * @mm: mm context of the victim 5465 * @gfp_mask: reclaim mode 5466 * @memcgp: charged memcg return 5467 * @compound: charge the page as compound or small page 5468 * 5469 * Try to charge @page to the memcg that @mm belongs to, reclaiming 5470 * pages according to @gfp_mask if necessary. 5471 * 5472 * Returns 0 on success, with *@memcgp pointing to the charged memcg. 5473 * Otherwise, an error code is returned. 5474 * 5475 * After page->mapping has been set up, the caller must finalize the 5476 * charge with mem_cgroup_commit_charge(). Or abort the transaction 5477 * with mem_cgroup_cancel_charge() in case page instantiation fails. 5478 */ 5479 int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm, 5480 gfp_t gfp_mask, struct mem_cgroup **memcgp, 5481 bool compound) 5482 { 5483 struct mem_cgroup *memcg = NULL; 5484 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1; 5485 int ret = 0; 5486 5487 if (mem_cgroup_disabled()) 5488 goto out; 5489 5490 if (PageSwapCache(page)) { 5491 /* 5492 * Every swap fault against a single page tries to charge the 5493 * page, bail as early as possible. shmem_unuse() encounters 5494 * already charged pages, too. The USED bit is protected by 5495 * the page lock, which serializes swap cache removal, which 5496 * in turn serializes uncharging. 5497 */ 5498 VM_BUG_ON_PAGE(!PageLocked(page), page); 5499 if (compound_head(page)->mem_cgroup) 5500 goto out; 5501 5502 if (do_swap_account) { 5503 swp_entry_t ent = { .val = page_private(page), }; 5504 unsigned short id = lookup_swap_cgroup_id(ent); 5505 5506 rcu_read_lock(); 5507 memcg = mem_cgroup_from_id(id); 5508 if (memcg && !css_tryget_online(&memcg->css)) 5509 memcg = NULL; 5510 rcu_read_unlock(); 5511 } 5512 } 5513 5514 if (!memcg) 5515 memcg = get_mem_cgroup_from_mm(mm); 5516 5517 ret = try_charge(memcg, gfp_mask, nr_pages); 5518 5519 css_put(&memcg->css); 5520 out: 5521 *memcgp = memcg; 5522 return ret; 5523 } 5524 5525 /** 5526 * mem_cgroup_commit_charge - commit a page charge 5527 * @page: page to charge 5528 * @memcg: memcg to charge the page to 5529 * @lrucare: page might be on LRU already 5530 * @compound: charge the page as compound or small page 5531 * 5532 * Finalize a charge transaction started by mem_cgroup_try_charge(), 5533 * after page->mapping has been set up. This must happen atomically 5534 * as part of the page instantiation, i.e. under the page table lock 5535 * for anonymous pages, under the page lock for page and swap cache. 5536 * 5537 * In addition, the page must not be on the LRU during the commit, to 5538 * prevent racing with task migration. If it might be, use @lrucare. 5539 * 5540 * Use mem_cgroup_cancel_charge() to cancel the transaction instead. 5541 */ 5542 void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg, 5543 bool lrucare, bool compound) 5544 { 5545 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1; 5546 5547 VM_BUG_ON_PAGE(!page->mapping, page); 5548 VM_BUG_ON_PAGE(PageLRU(page) && !lrucare, page); 5549 5550 if (mem_cgroup_disabled()) 5551 return; 5552 /* 5553 * Swap faults will attempt to charge the same page multiple 5554 * times. But reuse_swap_page() might have removed the page 5555 * from swapcache already, so we can't check PageSwapCache(). 5556 */ 5557 if (!memcg) 5558 return; 5559 5560 commit_charge(page, memcg, lrucare); 5561 5562 local_irq_disable(); 5563 mem_cgroup_charge_statistics(memcg, page, compound, nr_pages); 5564 memcg_check_events(memcg, page); 5565 local_irq_enable(); 5566 5567 if (do_memsw_account() && PageSwapCache(page)) { 5568 swp_entry_t entry = { .val = page_private(page) }; 5569 /* 5570 * The swap entry might not get freed for a long time, 5571 * let's not wait for it. The page already received a 5572 * memory+swap charge, drop the swap entry duplicate. 5573 */ 5574 mem_cgroup_uncharge_swap(entry, nr_pages); 5575 } 5576 } 5577 5578 /** 5579 * mem_cgroup_cancel_charge - cancel a page charge 5580 * @page: page to charge 5581 * @memcg: memcg to charge the page to 5582 * @compound: charge the page as compound or small page 5583 * 5584 * Cancel a charge transaction started by mem_cgroup_try_charge(). 5585 */ 5586 void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg, 5587 bool compound) 5588 { 5589 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1; 5590 5591 if (mem_cgroup_disabled()) 5592 return; 5593 /* 5594 * Swap faults will attempt to charge the same page multiple 5595 * times. But reuse_swap_page() might have removed the page 5596 * from swapcache already, so we can't check PageSwapCache(). 5597 */ 5598 if (!memcg) 5599 return; 5600 5601 cancel_charge(memcg, nr_pages); 5602 } 5603 5604 struct uncharge_gather { 5605 struct mem_cgroup *memcg; 5606 unsigned long pgpgout; 5607 unsigned long nr_anon; 5608 unsigned long nr_file; 5609 unsigned long nr_kmem; 5610 unsigned long nr_huge; 5611 unsigned long nr_shmem; 5612 struct page *dummy_page; 5613 }; 5614 5615 static inline void uncharge_gather_clear(struct uncharge_gather *ug) 5616 { 5617 memset(ug, 0, sizeof(*ug)); 5618 } 5619 5620 static void uncharge_batch(const struct uncharge_gather *ug) 5621 { 5622 unsigned long nr_pages = ug->nr_anon + ug->nr_file + ug->nr_kmem; 5623 unsigned long flags; 5624 5625 if (!mem_cgroup_is_root(ug->memcg)) { 5626 page_counter_uncharge(&ug->memcg->memory, nr_pages); 5627 if (do_memsw_account()) 5628 page_counter_uncharge(&ug->memcg->memsw, nr_pages); 5629 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && ug->nr_kmem) 5630 page_counter_uncharge(&ug->memcg->kmem, ug->nr_kmem); 5631 memcg_oom_recover(ug->memcg); 5632 } 5633 5634 local_irq_save(flags); 5635 __this_cpu_sub(ug->memcg->stat->count[MEMCG_RSS], ug->nr_anon); 5636 __this_cpu_sub(ug->memcg->stat->count[MEMCG_CACHE], ug->nr_file); 5637 __this_cpu_sub(ug->memcg->stat->count[MEMCG_RSS_HUGE], ug->nr_huge); 5638 __this_cpu_sub(ug->memcg->stat->count[NR_SHMEM], ug->nr_shmem); 5639 __this_cpu_add(ug->memcg->stat->events[PGPGOUT], ug->pgpgout); 5640 __this_cpu_add(ug->memcg->stat->nr_page_events, nr_pages); 5641 memcg_check_events(ug->memcg, ug->dummy_page); 5642 local_irq_restore(flags); 5643 5644 if (!mem_cgroup_is_root(ug->memcg)) 5645 css_put_many(&ug->memcg->css, nr_pages); 5646 } 5647 5648 static void uncharge_page(struct page *page, struct uncharge_gather *ug) 5649 { 5650 VM_BUG_ON_PAGE(PageLRU(page), page); 5651 VM_BUG_ON_PAGE(!PageHWPoison(page) && page_count(page), page); 5652 5653 if (!page->mem_cgroup) 5654 return; 5655 5656 /* 5657 * Nobody should be changing or seriously looking at 5658 * page->mem_cgroup at this point, we have fully 5659 * exclusive access to the page. 5660 */ 5661 5662 if (ug->memcg != page->mem_cgroup) { 5663 if (ug->memcg) { 5664 uncharge_batch(ug); 5665 uncharge_gather_clear(ug); 5666 } 5667 ug->memcg = page->mem_cgroup; 5668 } 5669 5670 if (!PageKmemcg(page)) { 5671 unsigned int nr_pages = 1; 5672 5673 if (PageTransHuge(page)) { 5674 nr_pages <<= compound_order(page); 5675 ug->nr_huge += nr_pages; 5676 } 5677 if (PageAnon(page)) 5678 ug->nr_anon += nr_pages; 5679 else { 5680 ug->nr_file += nr_pages; 5681 if (PageSwapBacked(page)) 5682 ug->nr_shmem += nr_pages; 5683 } 5684 ug->pgpgout++; 5685 } else { 5686 ug->nr_kmem += 1 << compound_order(page); 5687 __ClearPageKmemcg(page); 5688 } 5689 5690 ug->dummy_page = page; 5691 page->mem_cgroup = NULL; 5692 } 5693 5694 static void uncharge_list(struct list_head *page_list) 5695 { 5696 struct uncharge_gather ug; 5697 struct list_head *next; 5698 5699 uncharge_gather_clear(&ug); 5700 5701 /* 5702 * Note that the list can be a single page->lru; hence the 5703 * do-while loop instead of a simple list_for_each_entry(). 5704 */ 5705 next = page_list->next; 5706 do { 5707 struct page *page; 5708 5709 page = list_entry(next, struct page, lru); 5710 next = page->lru.next; 5711 5712 uncharge_page(page, &ug); 5713 } while (next != page_list); 5714 5715 if (ug.memcg) 5716 uncharge_batch(&ug); 5717 } 5718 5719 /** 5720 * mem_cgroup_uncharge - uncharge a page 5721 * @page: page to uncharge 5722 * 5723 * Uncharge a page previously charged with mem_cgroup_try_charge() and 5724 * mem_cgroup_commit_charge(). 5725 */ 5726 void mem_cgroup_uncharge(struct page *page) 5727 { 5728 struct uncharge_gather ug; 5729 5730 if (mem_cgroup_disabled()) 5731 return; 5732 5733 /* Don't touch page->lru of any random page, pre-check: */ 5734 if (!page->mem_cgroup) 5735 return; 5736 5737 uncharge_gather_clear(&ug); 5738 uncharge_page(page, &ug); 5739 uncharge_batch(&ug); 5740 } 5741 5742 /** 5743 * mem_cgroup_uncharge_list - uncharge a list of page 5744 * @page_list: list of pages to uncharge 5745 * 5746 * Uncharge a list of pages previously charged with 5747 * mem_cgroup_try_charge() and mem_cgroup_commit_charge(). 5748 */ 5749 void mem_cgroup_uncharge_list(struct list_head *page_list) 5750 { 5751 if (mem_cgroup_disabled()) 5752 return; 5753 5754 if (!list_empty(page_list)) 5755 uncharge_list(page_list); 5756 } 5757 5758 /** 5759 * mem_cgroup_migrate - charge a page's replacement 5760 * @oldpage: currently circulating page 5761 * @newpage: replacement page 5762 * 5763 * Charge @newpage as a replacement page for @oldpage. @oldpage will 5764 * be uncharged upon free. 5765 * 5766 * Both pages must be locked, @newpage->mapping must be set up. 5767 */ 5768 void mem_cgroup_migrate(struct page *oldpage, struct page *newpage) 5769 { 5770 struct mem_cgroup *memcg; 5771 unsigned int nr_pages; 5772 bool compound; 5773 unsigned long flags; 5774 5775 VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage); 5776 VM_BUG_ON_PAGE(!PageLocked(newpage), newpage); 5777 VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage); 5778 VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage), 5779 newpage); 5780 5781 if (mem_cgroup_disabled()) 5782 return; 5783 5784 /* Page cache replacement: new page already charged? */ 5785 if (newpage->mem_cgroup) 5786 return; 5787 5788 /* Swapcache readahead pages can get replaced before being charged */ 5789 memcg = oldpage->mem_cgroup; 5790 if (!memcg) 5791 return; 5792 5793 /* Force-charge the new page. The old one will be freed soon */ 5794 compound = PageTransHuge(newpage); 5795 nr_pages = compound ? hpage_nr_pages(newpage) : 1; 5796 5797 page_counter_charge(&memcg->memory, nr_pages); 5798 if (do_memsw_account()) 5799 page_counter_charge(&memcg->memsw, nr_pages); 5800 css_get_many(&memcg->css, nr_pages); 5801 5802 commit_charge(newpage, memcg, false); 5803 5804 local_irq_save(flags); 5805 mem_cgroup_charge_statistics(memcg, newpage, compound, nr_pages); 5806 memcg_check_events(memcg, newpage); 5807 local_irq_restore(flags); 5808 } 5809 5810 DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key); 5811 EXPORT_SYMBOL(memcg_sockets_enabled_key); 5812 5813 void mem_cgroup_sk_alloc(struct sock *sk) 5814 { 5815 struct mem_cgroup *memcg; 5816 5817 if (!mem_cgroup_sockets_enabled) 5818 return; 5819 5820 /* 5821 * Socket cloning can throw us here with sk_memcg already 5822 * filled. It won't however, necessarily happen from 5823 * process context. So the test for root memcg given 5824 * the current task's memcg won't help us in this case. 5825 * 5826 * Respecting the original socket's memcg is a better 5827 * decision in this case. 5828 */ 5829 if (sk->sk_memcg) { 5830 BUG_ON(mem_cgroup_is_root(sk->sk_memcg)); 5831 css_get(&sk->sk_memcg->css); 5832 return; 5833 } 5834 5835 rcu_read_lock(); 5836 memcg = mem_cgroup_from_task(current); 5837 if (memcg == root_mem_cgroup) 5838 goto out; 5839 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active) 5840 goto out; 5841 if (css_tryget_online(&memcg->css)) 5842 sk->sk_memcg = memcg; 5843 out: 5844 rcu_read_unlock(); 5845 } 5846 5847 void mem_cgroup_sk_free(struct sock *sk) 5848 { 5849 if (sk->sk_memcg) 5850 css_put(&sk->sk_memcg->css); 5851 } 5852 5853 /** 5854 * mem_cgroup_charge_skmem - charge socket memory 5855 * @memcg: memcg to charge 5856 * @nr_pages: number of pages to charge 5857 * 5858 * Charges @nr_pages to @memcg. Returns %true if the charge fit within 5859 * @memcg's configured limit, %false if the charge had to be forced. 5860 */ 5861 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages) 5862 { 5863 gfp_t gfp_mask = GFP_KERNEL; 5864 5865 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) { 5866 struct page_counter *fail; 5867 5868 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) { 5869 memcg->tcpmem_pressure = 0; 5870 return true; 5871 } 5872 page_counter_charge(&memcg->tcpmem, nr_pages); 5873 memcg->tcpmem_pressure = 1; 5874 return false; 5875 } 5876 5877 /* Don't block in the packet receive path */ 5878 if (in_softirq()) 5879 gfp_mask = GFP_NOWAIT; 5880 5881 this_cpu_add(memcg->stat->count[MEMCG_SOCK], nr_pages); 5882 5883 if (try_charge(memcg, gfp_mask, nr_pages) == 0) 5884 return true; 5885 5886 try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages); 5887 return false; 5888 } 5889 5890 /** 5891 * mem_cgroup_uncharge_skmem - uncharge socket memory 5892 * @memcg - memcg to uncharge 5893 * @nr_pages - number of pages to uncharge 5894 */ 5895 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages) 5896 { 5897 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) { 5898 page_counter_uncharge(&memcg->tcpmem, nr_pages); 5899 return; 5900 } 5901 5902 this_cpu_sub(memcg->stat->count[MEMCG_SOCK], nr_pages); 5903 5904 refill_stock(memcg, nr_pages); 5905 } 5906 5907 static int __init cgroup_memory(char *s) 5908 { 5909 char *token; 5910 5911 while ((token = strsep(&s, ",")) != NULL) { 5912 if (!*token) 5913 continue; 5914 if (!strcmp(token, "nosocket")) 5915 cgroup_memory_nosocket = true; 5916 if (!strcmp(token, "nokmem")) 5917 cgroup_memory_nokmem = true; 5918 } 5919 return 0; 5920 } 5921 __setup("cgroup.memory=", cgroup_memory); 5922 5923 /* 5924 * subsys_initcall() for memory controller. 5925 * 5926 * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this 5927 * context because of lock dependencies (cgroup_lock -> cpu hotplug) but 5928 * basically everything that doesn't depend on a specific mem_cgroup structure 5929 * should be initialized from here. 5930 */ 5931 static int __init mem_cgroup_init(void) 5932 { 5933 int cpu, node; 5934 5935 #ifndef CONFIG_SLOB 5936 /* 5937 * Kmem cache creation is mostly done with the slab_mutex held, 5938 * so use a workqueue with limited concurrency to avoid stalling 5939 * all worker threads in case lots of cgroups are created and 5940 * destroyed simultaneously. 5941 */ 5942 memcg_kmem_cache_wq = alloc_workqueue("memcg_kmem_cache", 0, 1); 5943 BUG_ON(!memcg_kmem_cache_wq); 5944 #endif 5945 5946 cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL, 5947 memcg_hotplug_cpu_dead); 5948 5949 for_each_possible_cpu(cpu) 5950 INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work, 5951 drain_local_stock); 5952 5953 for_each_node(node) { 5954 struct mem_cgroup_tree_per_node *rtpn; 5955 5956 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, 5957 node_online(node) ? node : NUMA_NO_NODE); 5958 5959 rtpn->rb_root = RB_ROOT; 5960 rtpn->rb_rightmost = NULL; 5961 spin_lock_init(&rtpn->lock); 5962 soft_limit_tree.rb_tree_per_node[node] = rtpn; 5963 } 5964 5965 return 0; 5966 } 5967 subsys_initcall(mem_cgroup_init); 5968 5969 #ifdef CONFIG_MEMCG_SWAP 5970 static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg) 5971 { 5972 while (!atomic_inc_not_zero(&memcg->id.ref)) { 5973 /* 5974 * The root cgroup cannot be destroyed, so it's refcount must 5975 * always be >= 1. 5976 */ 5977 if (WARN_ON_ONCE(memcg == root_mem_cgroup)) { 5978 VM_BUG_ON(1); 5979 break; 5980 } 5981 memcg = parent_mem_cgroup(memcg); 5982 if (!memcg) 5983 memcg = root_mem_cgroup; 5984 } 5985 return memcg; 5986 } 5987 5988 /** 5989 * mem_cgroup_swapout - transfer a memsw charge to swap 5990 * @page: page whose memsw charge to transfer 5991 * @entry: swap entry to move the charge to 5992 * 5993 * Transfer the memsw charge of @page to @entry. 5994 */ 5995 void mem_cgroup_swapout(struct page *page, swp_entry_t entry) 5996 { 5997 struct mem_cgroup *memcg, *swap_memcg; 5998 unsigned int nr_entries; 5999 unsigned short oldid; 6000 6001 VM_BUG_ON_PAGE(PageLRU(page), page); 6002 VM_BUG_ON_PAGE(page_count(page), page); 6003 6004 if (!do_memsw_account()) 6005 return; 6006 6007 memcg = page->mem_cgroup; 6008 6009 /* Readahead page, never charged */ 6010 if (!memcg) 6011 return; 6012 6013 /* 6014 * In case the memcg owning these pages has been offlined and doesn't 6015 * have an ID allocated to it anymore, charge the closest online 6016 * ancestor for the swap instead and transfer the memory+swap charge. 6017 */ 6018 swap_memcg = mem_cgroup_id_get_online(memcg); 6019 nr_entries = hpage_nr_pages(page); 6020 /* Get references for the tail pages, too */ 6021 if (nr_entries > 1) 6022 mem_cgroup_id_get_many(swap_memcg, nr_entries - 1); 6023 oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg), 6024 nr_entries); 6025 VM_BUG_ON_PAGE(oldid, page); 6026 mem_cgroup_swap_statistics(swap_memcg, nr_entries); 6027 6028 page->mem_cgroup = NULL; 6029 6030 if (!mem_cgroup_is_root(memcg)) 6031 page_counter_uncharge(&memcg->memory, nr_entries); 6032 6033 if (memcg != swap_memcg) { 6034 if (!mem_cgroup_is_root(swap_memcg)) 6035 page_counter_charge(&swap_memcg->memsw, nr_entries); 6036 page_counter_uncharge(&memcg->memsw, nr_entries); 6037 } 6038 6039 /* 6040 * Interrupts should be disabled here because the caller holds the 6041 * mapping->tree_lock lock which is taken with interrupts-off. It is 6042 * important here to have the interrupts disabled because it is the 6043 * only synchronisation we have for udpating the per-CPU variables. 6044 */ 6045 VM_BUG_ON(!irqs_disabled()); 6046 mem_cgroup_charge_statistics(memcg, page, PageTransHuge(page), 6047 -nr_entries); 6048 memcg_check_events(memcg, page); 6049 6050 if (!mem_cgroup_is_root(memcg)) 6051 css_put(&memcg->css); 6052 } 6053 6054 /** 6055 * mem_cgroup_try_charge_swap - try charging swap space for a page 6056 * @page: page being added to swap 6057 * @entry: swap entry to charge 6058 * 6059 * Try to charge @page's memcg for the swap space at @entry. 6060 * 6061 * Returns 0 on success, -ENOMEM on failure. 6062 */ 6063 int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry) 6064 { 6065 unsigned int nr_pages = hpage_nr_pages(page); 6066 struct page_counter *counter; 6067 struct mem_cgroup *memcg; 6068 unsigned short oldid; 6069 6070 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) || !do_swap_account) 6071 return 0; 6072 6073 memcg = page->mem_cgroup; 6074 6075 /* Readahead page, never charged */ 6076 if (!memcg) 6077 return 0; 6078 6079 memcg = mem_cgroup_id_get_online(memcg); 6080 6081 if (!mem_cgroup_is_root(memcg) && 6082 !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) { 6083 mem_cgroup_id_put(memcg); 6084 return -ENOMEM; 6085 } 6086 6087 /* Get references for the tail pages, too */ 6088 if (nr_pages > 1) 6089 mem_cgroup_id_get_many(memcg, nr_pages - 1); 6090 oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages); 6091 VM_BUG_ON_PAGE(oldid, page); 6092 mem_cgroup_swap_statistics(memcg, nr_pages); 6093 6094 return 0; 6095 } 6096 6097 /** 6098 * mem_cgroup_uncharge_swap - uncharge swap space 6099 * @entry: swap entry to uncharge 6100 * @nr_pages: the amount of swap space to uncharge 6101 */ 6102 void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages) 6103 { 6104 struct mem_cgroup *memcg; 6105 unsigned short id; 6106 6107 if (!do_swap_account) 6108 return; 6109 6110 id = swap_cgroup_record(entry, 0, nr_pages); 6111 rcu_read_lock(); 6112 memcg = mem_cgroup_from_id(id); 6113 if (memcg) { 6114 if (!mem_cgroup_is_root(memcg)) { 6115 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 6116 page_counter_uncharge(&memcg->swap, nr_pages); 6117 else 6118 page_counter_uncharge(&memcg->memsw, nr_pages); 6119 } 6120 mem_cgroup_swap_statistics(memcg, -nr_pages); 6121 mem_cgroup_id_put_many(memcg, nr_pages); 6122 } 6123 rcu_read_unlock(); 6124 } 6125 6126 long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg) 6127 { 6128 long nr_swap_pages = get_nr_swap_pages(); 6129 6130 if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys)) 6131 return nr_swap_pages; 6132 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) 6133 nr_swap_pages = min_t(long, nr_swap_pages, 6134 READ_ONCE(memcg->swap.limit) - 6135 page_counter_read(&memcg->swap)); 6136 return nr_swap_pages; 6137 } 6138 6139 bool mem_cgroup_swap_full(struct page *page) 6140 { 6141 struct mem_cgroup *memcg; 6142 6143 VM_BUG_ON_PAGE(!PageLocked(page), page); 6144 6145 if (vm_swap_full()) 6146 return true; 6147 if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys)) 6148 return false; 6149 6150 memcg = page->mem_cgroup; 6151 if (!memcg) 6152 return false; 6153 6154 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) 6155 if (page_counter_read(&memcg->swap) * 2 >= memcg->swap.limit) 6156 return true; 6157 6158 return false; 6159 } 6160 6161 /* for remember boot option*/ 6162 #ifdef CONFIG_MEMCG_SWAP_ENABLED 6163 static int really_do_swap_account __initdata = 1; 6164 #else 6165 static int really_do_swap_account __initdata; 6166 #endif 6167 6168 static int __init enable_swap_account(char *s) 6169 { 6170 if (!strcmp(s, "1")) 6171 really_do_swap_account = 1; 6172 else if (!strcmp(s, "0")) 6173 really_do_swap_account = 0; 6174 return 1; 6175 } 6176 __setup("swapaccount=", enable_swap_account); 6177 6178 static u64 swap_current_read(struct cgroup_subsys_state *css, 6179 struct cftype *cft) 6180 { 6181 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 6182 6183 return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE; 6184 } 6185 6186 static int swap_max_show(struct seq_file *m, void *v) 6187 { 6188 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 6189 unsigned long max = READ_ONCE(memcg->swap.limit); 6190 6191 if (max == PAGE_COUNTER_MAX) 6192 seq_puts(m, "max\n"); 6193 else 6194 seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE); 6195 6196 return 0; 6197 } 6198 6199 static ssize_t swap_max_write(struct kernfs_open_file *of, 6200 char *buf, size_t nbytes, loff_t off) 6201 { 6202 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 6203 unsigned long max; 6204 int err; 6205 6206 buf = strstrip(buf); 6207 err = page_counter_memparse(buf, "max", &max); 6208 if (err) 6209 return err; 6210 6211 mutex_lock(&memcg_limit_mutex); 6212 err = page_counter_limit(&memcg->swap, max); 6213 mutex_unlock(&memcg_limit_mutex); 6214 if (err) 6215 return err; 6216 6217 return nbytes; 6218 } 6219 6220 static struct cftype swap_files[] = { 6221 { 6222 .name = "swap.current", 6223 .flags = CFTYPE_NOT_ON_ROOT, 6224 .read_u64 = swap_current_read, 6225 }, 6226 { 6227 .name = "swap.max", 6228 .flags = CFTYPE_NOT_ON_ROOT, 6229 .seq_show = swap_max_show, 6230 .write = swap_max_write, 6231 }, 6232 { } /* terminate */ 6233 }; 6234 6235 static struct cftype memsw_cgroup_files[] = { 6236 { 6237 .name = "memsw.usage_in_bytes", 6238 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE), 6239 .read_u64 = mem_cgroup_read_u64, 6240 }, 6241 { 6242 .name = "memsw.max_usage_in_bytes", 6243 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE), 6244 .write = mem_cgroup_reset, 6245 .read_u64 = mem_cgroup_read_u64, 6246 }, 6247 { 6248 .name = "memsw.limit_in_bytes", 6249 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT), 6250 .write = mem_cgroup_write, 6251 .read_u64 = mem_cgroup_read_u64, 6252 }, 6253 { 6254 .name = "memsw.failcnt", 6255 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT), 6256 .write = mem_cgroup_reset, 6257 .read_u64 = mem_cgroup_read_u64, 6258 }, 6259 { }, /* terminate */ 6260 }; 6261 6262 static int __init mem_cgroup_swap_init(void) 6263 { 6264 if (!mem_cgroup_disabled() && really_do_swap_account) { 6265 do_swap_account = 1; 6266 WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, 6267 swap_files)); 6268 WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys, 6269 memsw_cgroup_files)); 6270 } 6271 return 0; 6272 } 6273 subsys_initcall(mem_cgroup_swap_init); 6274 6275 #endif /* CONFIG_MEMCG_SWAP */ 6276