1 /* memcontrol.c - Memory Controller 2 * 3 * Copyright IBM Corporation, 2007 4 * Author Balbir Singh <balbir@linux.vnet.ibm.com> 5 * 6 * Copyright 2007 OpenVZ SWsoft Inc 7 * Author: Pavel Emelianov <xemul@openvz.org> 8 * 9 * Memory thresholds 10 * Copyright (C) 2009 Nokia Corporation 11 * Author: Kirill A. Shutemov 12 * 13 * Kernel Memory Controller 14 * Copyright (C) 2012 Parallels Inc. and Google Inc. 15 * Authors: Glauber Costa and Suleiman Souhlal 16 * 17 * Native page reclaim 18 * Charge lifetime sanitation 19 * Lockless page tracking & accounting 20 * Unified hierarchy configuration model 21 * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner 22 * 23 * This program is free software; you can redistribute it and/or modify 24 * it under the terms of the GNU General Public License as published by 25 * the Free Software Foundation; either version 2 of the License, or 26 * (at your option) any later version. 27 * 28 * This program is distributed in the hope that it will be useful, 29 * but WITHOUT ANY WARRANTY; without even the implied warranty of 30 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 * GNU General Public License for more details. 32 */ 33 34 #include <linux/page_counter.h> 35 #include <linux/memcontrol.h> 36 #include <linux/cgroup.h> 37 #include <linux/mm.h> 38 #include <linux/hugetlb.h> 39 #include <linux/pagemap.h> 40 #include <linux/smp.h> 41 #include <linux/page-flags.h> 42 #include <linux/backing-dev.h> 43 #include <linux/bit_spinlock.h> 44 #include <linux/rcupdate.h> 45 #include <linux/limits.h> 46 #include <linux/export.h> 47 #include <linux/mutex.h> 48 #include <linux/rbtree.h> 49 #include <linux/slab.h> 50 #include <linux/swap.h> 51 #include <linux/swapops.h> 52 #include <linux/spinlock.h> 53 #include <linux/eventfd.h> 54 #include <linux/poll.h> 55 #include <linux/sort.h> 56 #include <linux/fs.h> 57 #include <linux/seq_file.h> 58 #include <linux/vmpressure.h> 59 #include <linux/mm_inline.h> 60 #include <linux/swap_cgroup.h> 61 #include <linux/cpu.h> 62 #include <linux/oom.h> 63 #include <linux/lockdep.h> 64 #include <linux/file.h> 65 #include <linux/tracehook.h> 66 #include "internal.h" 67 #include <net/sock.h> 68 #include <net/ip.h> 69 #include "slab.h" 70 71 #include <linux/uaccess.h> 72 73 #include <trace/events/vmscan.h> 74 75 struct cgroup_subsys memory_cgrp_subsys __read_mostly; 76 EXPORT_SYMBOL(memory_cgrp_subsys); 77 78 struct mem_cgroup *root_mem_cgroup __read_mostly; 79 80 #define MEM_CGROUP_RECLAIM_RETRIES 5 81 82 /* Socket memory accounting disabled? */ 83 static bool cgroup_memory_nosocket; 84 85 /* Kernel memory accounting disabled? */ 86 static bool cgroup_memory_nokmem; 87 88 /* Whether the swap controller is active */ 89 #ifdef CONFIG_MEMCG_SWAP 90 int do_swap_account __read_mostly; 91 #else 92 #define do_swap_account 0 93 #endif 94 95 /* Whether legacy memory+swap accounting is active */ 96 static bool do_memsw_account(void) 97 { 98 return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && do_swap_account; 99 } 100 101 static const char * const mem_cgroup_stat_names[] = { 102 "cache", 103 "rss", 104 "rss_huge", 105 "mapped_file", 106 "dirty", 107 "writeback", 108 "swap", 109 }; 110 111 static const char * const mem_cgroup_events_names[] = { 112 "pgpgin", 113 "pgpgout", 114 "pgfault", 115 "pgmajfault", 116 }; 117 118 static const char * const mem_cgroup_lru_names[] = { 119 "inactive_anon", 120 "active_anon", 121 "inactive_file", 122 "active_file", 123 "unevictable", 124 }; 125 126 #define THRESHOLDS_EVENTS_TARGET 128 127 #define SOFTLIMIT_EVENTS_TARGET 1024 128 #define NUMAINFO_EVENTS_TARGET 1024 129 130 /* 131 * Cgroups above their limits are maintained in a RB-Tree, independent of 132 * their hierarchy representation 133 */ 134 135 struct mem_cgroup_tree_per_node { 136 struct rb_root rb_root; 137 spinlock_t lock; 138 }; 139 140 struct mem_cgroup_tree { 141 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES]; 142 }; 143 144 static struct mem_cgroup_tree soft_limit_tree __read_mostly; 145 146 /* for OOM */ 147 struct mem_cgroup_eventfd_list { 148 struct list_head list; 149 struct eventfd_ctx *eventfd; 150 }; 151 152 /* 153 * cgroup_event represents events which userspace want to receive. 154 */ 155 struct mem_cgroup_event { 156 /* 157 * memcg which the event belongs to. 158 */ 159 struct mem_cgroup *memcg; 160 /* 161 * eventfd to signal userspace about the event. 162 */ 163 struct eventfd_ctx *eventfd; 164 /* 165 * Each of these stored in a list by the cgroup. 166 */ 167 struct list_head list; 168 /* 169 * register_event() callback will be used to add new userspace 170 * waiter for changes related to this event. Use eventfd_signal() 171 * on eventfd to send notification to userspace. 172 */ 173 int (*register_event)(struct mem_cgroup *memcg, 174 struct eventfd_ctx *eventfd, const char *args); 175 /* 176 * unregister_event() callback will be called when userspace closes 177 * the eventfd or on cgroup removing. This callback must be set, 178 * if you want provide notification functionality. 179 */ 180 void (*unregister_event)(struct mem_cgroup *memcg, 181 struct eventfd_ctx *eventfd); 182 /* 183 * All fields below needed to unregister event when 184 * userspace closes eventfd. 185 */ 186 poll_table pt; 187 wait_queue_head_t *wqh; 188 wait_queue_t wait; 189 struct work_struct remove; 190 }; 191 192 static void mem_cgroup_threshold(struct mem_cgroup *memcg); 193 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg); 194 195 /* Stuffs for move charges at task migration. */ 196 /* 197 * Types of charges to be moved. 198 */ 199 #define MOVE_ANON 0x1U 200 #define MOVE_FILE 0x2U 201 #define MOVE_MASK (MOVE_ANON | MOVE_FILE) 202 203 /* "mc" and its members are protected by cgroup_mutex */ 204 static struct move_charge_struct { 205 spinlock_t lock; /* for from, to */ 206 struct mm_struct *mm; 207 struct mem_cgroup *from; 208 struct mem_cgroup *to; 209 unsigned long flags; 210 unsigned long precharge; 211 unsigned long moved_charge; 212 unsigned long moved_swap; 213 struct task_struct *moving_task; /* a task moving charges */ 214 wait_queue_head_t waitq; /* a waitq for other context */ 215 } mc = { 216 .lock = __SPIN_LOCK_UNLOCKED(mc.lock), 217 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq), 218 }; 219 220 /* 221 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft 222 * limit reclaim to prevent infinite loops, if they ever occur. 223 */ 224 #define MEM_CGROUP_MAX_RECLAIM_LOOPS 100 225 #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2 226 227 enum charge_type { 228 MEM_CGROUP_CHARGE_TYPE_CACHE = 0, 229 MEM_CGROUP_CHARGE_TYPE_ANON, 230 MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */ 231 MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */ 232 NR_CHARGE_TYPE, 233 }; 234 235 /* for encoding cft->private value on file */ 236 enum res_type { 237 _MEM, 238 _MEMSWAP, 239 _OOM_TYPE, 240 _KMEM, 241 _TCP, 242 }; 243 244 #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val)) 245 #define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff) 246 #define MEMFILE_ATTR(val) ((val) & 0xffff) 247 /* Used for OOM nofiier */ 248 #define OOM_CONTROL (0) 249 250 /* Some nice accessors for the vmpressure. */ 251 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg) 252 { 253 if (!memcg) 254 memcg = root_mem_cgroup; 255 return &memcg->vmpressure; 256 } 257 258 struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr) 259 { 260 return &container_of(vmpr, struct mem_cgroup, vmpressure)->css; 261 } 262 263 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg) 264 { 265 return (memcg == root_mem_cgroup); 266 } 267 268 #ifndef CONFIG_SLOB 269 /* 270 * This will be the memcg's index in each cache's ->memcg_params.memcg_caches. 271 * The main reason for not using cgroup id for this: 272 * this works better in sparse environments, where we have a lot of memcgs, 273 * but only a few kmem-limited. Or also, if we have, for instance, 200 274 * memcgs, and none but the 200th is kmem-limited, we'd have to have a 275 * 200 entry array for that. 276 * 277 * The current size of the caches array is stored in memcg_nr_cache_ids. It 278 * will double each time we have to increase it. 279 */ 280 static DEFINE_IDA(memcg_cache_ida); 281 int memcg_nr_cache_ids; 282 283 /* Protects memcg_nr_cache_ids */ 284 static DECLARE_RWSEM(memcg_cache_ids_sem); 285 286 void memcg_get_cache_ids(void) 287 { 288 down_read(&memcg_cache_ids_sem); 289 } 290 291 void memcg_put_cache_ids(void) 292 { 293 up_read(&memcg_cache_ids_sem); 294 } 295 296 /* 297 * MIN_SIZE is different than 1, because we would like to avoid going through 298 * the alloc/free process all the time. In a small machine, 4 kmem-limited 299 * cgroups is a reasonable guess. In the future, it could be a parameter or 300 * tunable, but that is strictly not necessary. 301 * 302 * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get 303 * this constant directly from cgroup, but it is understandable that this is 304 * better kept as an internal representation in cgroup.c. In any case, the 305 * cgrp_id space is not getting any smaller, and we don't have to necessarily 306 * increase ours as well if it increases. 307 */ 308 #define MEMCG_CACHES_MIN_SIZE 4 309 #define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX 310 311 /* 312 * A lot of the calls to the cache allocation functions are expected to be 313 * inlined by the compiler. Since the calls to memcg_kmem_get_cache are 314 * conditional to this static branch, we'll have to allow modules that does 315 * kmem_cache_alloc and the such to see this symbol as well 316 */ 317 DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key); 318 EXPORT_SYMBOL(memcg_kmem_enabled_key); 319 320 #endif /* !CONFIG_SLOB */ 321 322 /** 323 * mem_cgroup_css_from_page - css of the memcg associated with a page 324 * @page: page of interest 325 * 326 * If memcg is bound to the default hierarchy, css of the memcg associated 327 * with @page is returned. The returned css remains associated with @page 328 * until it is released. 329 * 330 * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup 331 * is returned. 332 */ 333 struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page) 334 { 335 struct mem_cgroup *memcg; 336 337 memcg = page->mem_cgroup; 338 339 if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys)) 340 memcg = root_mem_cgroup; 341 342 return &memcg->css; 343 } 344 345 /** 346 * page_cgroup_ino - return inode number of the memcg a page is charged to 347 * @page: the page 348 * 349 * Look up the closest online ancestor of the memory cgroup @page is charged to 350 * and return its inode number or 0 if @page is not charged to any cgroup. It 351 * is safe to call this function without holding a reference to @page. 352 * 353 * Note, this function is inherently racy, because there is nothing to prevent 354 * the cgroup inode from getting torn down and potentially reallocated a moment 355 * after page_cgroup_ino() returns, so it only should be used by callers that 356 * do not care (such as procfs interfaces). 357 */ 358 ino_t page_cgroup_ino(struct page *page) 359 { 360 struct mem_cgroup *memcg; 361 unsigned long ino = 0; 362 363 rcu_read_lock(); 364 memcg = READ_ONCE(page->mem_cgroup); 365 while (memcg && !(memcg->css.flags & CSS_ONLINE)) 366 memcg = parent_mem_cgroup(memcg); 367 if (memcg) 368 ino = cgroup_ino(memcg->css.cgroup); 369 rcu_read_unlock(); 370 return ino; 371 } 372 373 static struct mem_cgroup_per_node * 374 mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page) 375 { 376 int nid = page_to_nid(page); 377 378 return memcg->nodeinfo[nid]; 379 } 380 381 static struct mem_cgroup_tree_per_node * 382 soft_limit_tree_node(int nid) 383 { 384 return soft_limit_tree.rb_tree_per_node[nid]; 385 } 386 387 static struct mem_cgroup_tree_per_node * 388 soft_limit_tree_from_page(struct page *page) 389 { 390 int nid = page_to_nid(page); 391 392 return soft_limit_tree.rb_tree_per_node[nid]; 393 } 394 395 static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz, 396 struct mem_cgroup_tree_per_node *mctz, 397 unsigned long new_usage_in_excess) 398 { 399 struct rb_node **p = &mctz->rb_root.rb_node; 400 struct rb_node *parent = NULL; 401 struct mem_cgroup_per_node *mz_node; 402 403 if (mz->on_tree) 404 return; 405 406 mz->usage_in_excess = new_usage_in_excess; 407 if (!mz->usage_in_excess) 408 return; 409 while (*p) { 410 parent = *p; 411 mz_node = rb_entry(parent, struct mem_cgroup_per_node, 412 tree_node); 413 if (mz->usage_in_excess < mz_node->usage_in_excess) 414 p = &(*p)->rb_left; 415 /* 416 * We can't avoid mem cgroups that are over their soft 417 * limit by the same amount 418 */ 419 else if (mz->usage_in_excess >= mz_node->usage_in_excess) 420 p = &(*p)->rb_right; 421 } 422 rb_link_node(&mz->tree_node, parent, p); 423 rb_insert_color(&mz->tree_node, &mctz->rb_root); 424 mz->on_tree = true; 425 } 426 427 static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz, 428 struct mem_cgroup_tree_per_node *mctz) 429 { 430 if (!mz->on_tree) 431 return; 432 rb_erase(&mz->tree_node, &mctz->rb_root); 433 mz->on_tree = false; 434 } 435 436 static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz, 437 struct mem_cgroup_tree_per_node *mctz) 438 { 439 unsigned long flags; 440 441 spin_lock_irqsave(&mctz->lock, flags); 442 __mem_cgroup_remove_exceeded(mz, mctz); 443 spin_unlock_irqrestore(&mctz->lock, flags); 444 } 445 446 static unsigned long soft_limit_excess(struct mem_cgroup *memcg) 447 { 448 unsigned long nr_pages = page_counter_read(&memcg->memory); 449 unsigned long soft_limit = READ_ONCE(memcg->soft_limit); 450 unsigned long excess = 0; 451 452 if (nr_pages > soft_limit) 453 excess = nr_pages - soft_limit; 454 455 return excess; 456 } 457 458 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page) 459 { 460 unsigned long excess; 461 struct mem_cgroup_per_node *mz; 462 struct mem_cgroup_tree_per_node *mctz; 463 464 mctz = soft_limit_tree_from_page(page); 465 /* 466 * Necessary to update all ancestors when hierarchy is used. 467 * because their event counter is not touched. 468 */ 469 for (; memcg; memcg = parent_mem_cgroup(memcg)) { 470 mz = mem_cgroup_page_nodeinfo(memcg, page); 471 excess = soft_limit_excess(memcg); 472 /* 473 * We have to update the tree if mz is on RB-tree or 474 * mem is over its softlimit. 475 */ 476 if (excess || mz->on_tree) { 477 unsigned long flags; 478 479 spin_lock_irqsave(&mctz->lock, flags); 480 /* if on-tree, remove it */ 481 if (mz->on_tree) 482 __mem_cgroup_remove_exceeded(mz, mctz); 483 /* 484 * Insert again. mz->usage_in_excess will be updated. 485 * If excess is 0, no tree ops. 486 */ 487 __mem_cgroup_insert_exceeded(mz, mctz, excess); 488 spin_unlock_irqrestore(&mctz->lock, flags); 489 } 490 } 491 } 492 493 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg) 494 { 495 struct mem_cgroup_tree_per_node *mctz; 496 struct mem_cgroup_per_node *mz; 497 int nid; 498 499 for_each_node(nid) { 500 mz = mem_cgroup_nodeinfo(memcg, nid); 501 mctz = soft_limit_tree_node(nid); 502 mem_cgroup_remove_exceeded(mz, mctz); 503 } 504 } 505 506 static struct mem_cgroup_per_node * 507 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz) 508 { 509 struct rb_node *rightmost = NULL; 510 struct mem_cgroup_per_node *mz; 511 512 retry: 513 mz = NULL; 514 rightmost = rb_last(&mctz->rb_root); 515 if (!rightmost) 516 goto done; /* Nothing to reclaim from */ 517 518 mz = rb_entry(rightmost, struct mem_cgroup_per_node, tree_node); 519 /* 520 * Remove the node now but someone else can add it back, 521 * we will to add it back at the end of reclaim to its correct 522 * position in the tree. 523 */ 524 __mem_cgroup_remove_exceeded(mz, mctz); 525 if (!soft_limit_excess(mz->memcg) || 526 !css_tryget_online(&mz->memcg->css)) 527 goto retry; 528 done: 529 return mz; 530 } 531 532 static struct mem_cgroup_per_node * 533 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz) 534 { 535 struct mem_cgroup_per_node *mz; 536 537 spin_lock_irq(&mctz->lock); 538 mz = __mem_cgroup_largest_soft_limit_node(mctz); 539 spin_unlock_irq(&mctz->lock); 540 return mz; 541 } 542 543 /* 544 * Return page count for single (non recursive) @memcg. 545 * 546 * Implementation Note: reading percpu statistics for memcg. 547 * 548 * Both of vmstat[] and percpu_counter has threshold and do periodic 549 * synchronization to implement "quick" read. There are trade-off between 550 * reading cost and precision of value. Then, we may have a chance to implement 551 * a periodic synchronization of counter in memcg's counter. 552 * 553 * But this _read() function is used for user interface now. The user accounts 554 * memory usage by memory cgroup and he _always_ requires exact value because 555 * he accounts memory. Even if we provide quick-and-fuzzy read, we always 556 * have to visit all online cpus and make sum. So, for now, unnecessary 557 * synchronization is not implemented. (just implemented for cpu hotplug) 558 * 559 * If there are kernel internal actions which can make use of some not-exact 560 * value, and reading all cpu value can be performance bottleneck in some 561 * common workload, threshold and synchronization as vmstat[] should be 562 * implemented. 563 */ 564 static unsigned long 565 mem_cgroup_read_stat(struct mem_cgroup *memcg, enum mem_cgroup_stat_index idx) 566 { 567 long val = 0; 568 int cpu; 569 570 /* Per-cpu values can be negative, use a signed accumulator */ 571 for_each_possible_cpu(cpu) 572 val += per_cpu(memcg->stat->count[idx], cpu); 573 /* 574 * Summing races with updates, so val may be negative. Avoid exposing 575 * transient negative values. 576 */ 577 if (val < 0) 578 val = 0; 579 return val; 580 } 581 582 static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg, 583 enum mem_cgroup_events_index idx) 584 { 585 unsigned long val = 0; 586 int cpu; 587 588 for_each_possible_cpu(cpu) 589 val += per_cpu(memcg->stat->events[idx], cpu); 590 return val; 591 } 592 593 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg, 594 struct page *page, 595 bool compound, int nr_pages) 596 { 597 /* 598 * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is 599 * counted as CACHE even if it's on ANON LRU. 600 */ 601 if (PageAnon(page)) 602 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS], 603 nr_pages); 604 else 605 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE], 606 nr_pages); 607 608 if (compound) { 609 VM_BUG_ON_PAGE(!PageTransHuge(page), page); 610 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], 611 nr_pages); 612 } 613 614 /* pagein of a big page is an event. So, ignore page size */ 615 if (nr_pages > 0) 616 __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]); 617 else { 618 __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]); 619 nr_pages = -nr_pages; /* for event */ 620 } 621 622 __this_cpu_add(memcg->stat->nr_page_events, nr_pages); 623 } 624 625 unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg, 626 int nid, unsigned int lru_mask) 627 { 628 unsigned long nr = 0; 629 struct mem_cgroup_per_node *mz; 630 enum lru_list lru; 631 632 VM_BUG_ON((unsigned)nid >= nr_node_ids); 633 634 for_each_lru(lru) { 635 if (!(BIT(lru) & lru_mask)) 636 continue; 637 mz = mem_cgroup_nodeinfo(memcg, nid); 638 nr += mz->lru_size[lru]; 639 } 640 return nr; 641 } 642 643 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg, 644 unsigned int lru_mask) 645 { 646 unsigned long nr = 0; 647 int nid; 648 649 for_each_node_state(nid, N_MEMORY) 650 nr += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask); 651 return nr; 652 } 653 654 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg, 655 enum mem_cgroup_events_target target) 656 { 657 unsigned long val, next; 658 659 val = __this_cpu_read(memcg->stat->nr_page_events); 660 next = __this_cpu_read(memcg->stat->targets[target]); 661 /* from time_after() in jiffies.h */ 662 if ((long)next - (long)val < 0) { 663 switch (target) { 664 case MEM_CGROUP_TARGET_THRESH: 665 next = val + THRESHOLDS_EVENTS_TARGET; 666 break; 667 case MEM_CGROUP_TARGET_SOFTLIMIT: 668 next = val + SOFTLIMIT_EVENTS_TARGET; 669 break; 670 case MEM_CGROUP_TARGET_NUMAINFO: 671 next = val + NUMAINFO_EVENTS_TARGET; 672 break; 673 default: 674 break; 675 } 676 __this_cpu_write(memcg->stat->targets[target], next); 677 return true; 678 } 679 return false; 680 } 681 682 /* 683 * Check events in order. 684 * 685 */ 686 static void memcg_check_events(struct mem_cgroup *memcg, struct page *page) 687 { 688 /* threshold event is triggered in finer grain than soft limit */ 689 if (unlikely(mem_cgroup_event_ratelimit(memcg, 690 MEM_CGROUP_TARGET_THRESH))) { 691 bool do_softlimit; 692 bool do_numainfo __maybe_unused; 693 694 do_softlimit = mem_cgroup_event_ratelimit(memcg, 695 MEM_CGROUP_TARGET_SOFTLIMIT); 696 #if MAX_NUMNODES > 1 697 do_numainfo = mem_cgroup_event_ratelimit(memcg, 698 MEM_CGROUP_TARGET_NUMAINFO); 699 #endif 700 mem_cgroup_threshold(memcg); 701 if (unlikely(do_softlimit)) 702 mem_cgroup_update_tree(memcg, page); 703 #if MAX_NUMNODES > 1 704 if (unlikely(do_numainfo)) 705 atomic_inc(&memcg->numainfo_events); 706 #endif 707 } 708 } 709 710 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) 711 { 712 /* 713 * mm_update_next_owner() may clear mm->owner to NULL 714 * if it races with swapoff, page migration, etc. 715 * So this can be called with p == NULL. 716 */ 717 if (unlikely(!p)) 718 return NULL; 719 720 return mem_cgroup_from_css(task_css(p, memory_cgrp_id)); 721 } 722 EXPORT_SYMBOL(mem_cgroup_from_task); 723 724 static struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm) 725 { 726 struct mem_cgroup *memcg = NULL; 727 728 rcu_read_lock(); 729 do { 730 /* 731 * Page cache insertions can happen withou an 732 * actual mm context, e.g. during disk probing 733 * on boot, loopback IO, acct() writes etc. 734 */ 735 if (unlikely(!mm)) 736 memcg = root_mem_cgroup; 737 else { 738 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); 739 if (unlikely(!memcg)) 740 memcg = root_mem_cgroup; 741 } 742 } while (!css_tryget_online(&memcg->css)); 743 rcu_read_unlock(); 744 return memcg; 745 } 746 747 /** 748 * mem_cgroup_iter - iterate over memory cgroup hierarchy 749 * @root: hierarchy root 750 * @prev: previously returned memcg, NULL on first invocation 751 * @reclaim: cookie for shared reclaim walks, NULL for full walks 752 * 753 * Returns references to children of the hierarchy below @root, or 754 * @root itself, or %NULL after a full round-trip. 755 * 756 * Caller must pass the return value in @prev on subsequent 757 * invocations for reference counting, or use mem_cgroup_iter_break() 758 * to cancel a hierarchy walk before the round-trip is complete. 759 * 760 * Reclaimers can specify a zone and a priority level in @reclaim to 761 * divide up the memcgs in the hierarchy among all concurrent 762 * reclaimers operating on the same zone and priority. 763 */ 764 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root, 765 struct mem_cgroup *prev, 766 struct mem_cgroup_reclaim_cookie *reclaim) 767 { 768 struct mem_cgroup_reclaim_iter *uninitialized_var(iter); 769 struct cgroup_subsys_state *css = NULL; 770 struct mem_cgroup *memcg = NULL; 771 struct mem_cgroup *pos = NULL; 772 773 if (mem_cgroup_disabled()) 774 return NULL; 775 776 if (!root) 777 root = root_mem_cgroup; 778 779 if (prev && !reclaim) 780 pos = prev; 781 782 if (!root->use_hierarchy && root != root_mem_cgroup) { 783 if (prev) 784 goto out; 785 return root; 786 } 787 788 rcu_read_lock(); 789 790 if (reclaim) { 791 struct mem_cgroup_per_node *mz; 792 793 mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id); 794 iter = &mz->iter[reclaim->priority]; 795 796 if (prev && reclaim->generation != iter->generation) 797 goto out_unlock; 798 799 while (1) { 800 pos = READ_ONCE(iter->position); 801 if (!pos || css_tryget(&pos->css)) 802 break; 803 /* 804 * css reference reached zero, so iter->position will 805 * be cleared by ->css_released. However, we should not 806 * rely on this happening soon, because ->css_released 807 * is called from a work queue, and by busy-waiting we 808 * might block it. So we clear iter->position right 809 * away. 810 */ 811 (void)cmpxchg(&iter->position, pos, NULL); 812 } 813 } 814 815 if (pos) 816 css = &pos->css; 817 818 for (;;) { 819 css = css_next_descendant_pre(css, &root->css); 820 if (!css) { 821 /* 822 * Reclaimers share the hierarchy walk, and a 823 * new one might jump in right at the end of 824 * the hierarchy - make sure they see at least 825 * one group and restart from the beginning. 826 */ 827 if (!prev) 828 continue; 829 break; 830 } 831 832 /* 833 * Verify the css and acquire a reference. The root 834 * is provided by the caller, so we know it's alive 835 * and kicking, and don't take an extra reference. 836 */ 837 memcg = mem_cgroup_from_css(css); 838 839 if (css == &root->css) 840 break; 841 842 if (css_tryget(css)) 843 break; 844 845 memcg = NULL; 846 } 847 848 if (reclaim) { 849 /* 850 * The position could have already been updated by a competing 851 * thread, so check that the value hasn't changed since we read 852 * it to avoid reclaiming from the same cgroup twice. 853 */ 854 (void)cmpxchg(&iter->position, pos, memcg); 855 856 if (pos) 857 css_put(&pos->css); 858 859 if (!memcg) 860 iter->generation++; 861 else if (!prev) 862 reclaim->generation = iter->generation; 863 } 864 865 out_unlock: 866 rcu_read_unlock(); 867 out: 868 if (prev && prev != root) 869 css_put(&prev->css); 870 871 return memcg; 872 } 873 874 /** 875 * mem_cgroup_iter_break - abort a hierarchy walk prematurely 876 * @root: hierarchy root 877 * @prev: last visited hierarchy member as returned by mem_cgroup_iter() 878 */ 879 void mem_cgroup_iter_break(struct mem_cgroup *root, 880 struct mem_cgroup *prev) 881 { 882 if (!root) 883 root = root_mem_cgroup; 884 if (prev && prev != root) 885 css_put(&prev->css); 886 } 887 888 static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg) 889 { 890 struct mem_cgroup *memcg = dead_memcg; 891 struct mem_cgroup_reclaim_iter *iter; 892 struct mem_cgroup_per_node *mz; 893 int nid; 894 int i; 895 896 while ((memcg = parent_mem_cgroup(memcg))) { 897 for_each_node(nid) { 898 mz = mem_cgroup_nodeinfo(memcg, nid); 899 for (i = 0; i <= DEF_PRIORITY; i++) { 900 iter = &mz->iter[i]; 901 cmpxchg(&iter->position, 902 dead_memcg, NULL); 903 } 904 } 905 } 906 } 907 908 /* 909 * Iteration constructs for visiting all cgroups (under a tree). If 910 * loops are exited prematurely (break), mem_cgroup_iter_break() must 911 * be used for reference counting. 912 */ 913 #define for_each_mem_cgroup_tree(iter, root) \ 914 for (iter = mem_cgroup_iter(root, NULL, NULL); \ 915 iter != NULL; \ 916 iter = mem_cgroup_iter(root, iter, NULL)) 917 918 #define for_each_mem_cgroup(iter) \ 919 for (iter = mem_cgroup_iter(NULL, NULL, NULL); \ 920 iter != NULL; \ 921 iter = mem_cgroup_iter(NULL, iter, NULL)) 922 923 /** 924 * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy 925 * @memcg: hierarchy root 926 * @fn: function to call for each task 927 * @arg: argument passed to @fn 928 * 929 * This function iterates over tasks attached to @memcg or to any of its 930 * descendants and calls @fn for each task. If @fn returns a non-zero 931 * value, the function breaks the iteration loop and returns the value. 932 * Otherwise, it will iterate over all tasks and return 0. 933 * 934 * This function must not be called for the root memory cgroup. 935 */ 936 int mem_cgroup_scan_tasks(struct mem_cgroup *memcg, 937 int (*fn)(struct task_struct *, void *), void *arg) 938 { 939 struct mem_cgroup *iter; 940 int ret = 0; 941 942 BUG_ON(memcg == root_mem_cgroup); 943 944 for_each_mem_cgroup_tree(iter, memcg) { 945 struct css_task_iter it; 946 struct task_struct *task; 947 948 css_task_iter_start(&iter->css, &it); 949 while (!ret && (task = css_task_iter_next(&it))) 950 ret = fn(task, arg); 951 css_task_iter_end(&it); 952 if (ret) { 953 mem_cgroup_iter_break(memcg, iter); 954 break; 955 } 956 } 957 return ret; 958 } 959 960 /** 961 * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page 962 * @page: the page 963 * @zone: zone of the page 964 * 965 * This function is only safe when following the LRU page isolation 966 * and putback protocol: the LRU lock must be held, and the page must 967 * either be PageLRU() or the caller must have isolated/allocated it. 968 */ 969 struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgdat) 970 { 971 struct mem_cgroup_per_node *mz; 972 struct mem_cgroup *memcg; 973 struct lruvec *lruvec; 974 975 if (mem_cgroup_disabled()) { 976 lruvec = &pgdat->lruvec; 977 goto out; 978 } 979 980 memcg = page->mem_cgroup; 981 /* 982 * Swapcache readahead pages are added to the LRU - and 983 * possibly migrated - before they are charged. 984 */ 985 if (!memcg) 986 memcg = root_mem_cgroup; 987 988 mz = mem_cgroup_page_nodeinfo(memcg, page); 989 lruvec = &mz->lruvec; 990 out: 991 /* 992 * Since a node can be onlined after the mem_cgroup was created, 993 * we have to be prepared to initialize lruvec->zone here; 994 * and if offlined then reonlined, we need to reinitialize it. 995 */ 996 if (unlikely(lruvec->pgdat != pgdat)) 997 lruvec->pgdat = pgdat; 998 return lruvec; 999 } 1000 1001 /** 1002 * mem_cgroup_update_lru_size - account for adding or removing an lru page 1003 * @lruvec: mem_cgroup per zone lru vector 1004 * @lru: index of lru list the page is sitting on 1005 * @nr_pages: positive when adding or negative when removing 1006 * 1007 * This function must be called under lru_lock, just before a page is added 1008 * to or just after a page is removed from an lru list (that ordering being 1009 * so as to allow it to check that lru_size 0 is consistent with list_empty). 1010 */ 1011 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, 1012 int nr_pages) 1013 { 1014 struct mem_cgroup_per_node *mz; 1015 unsigned long *lru_size; 1016 long size; 1017 bool empty; 1018 1019 if (mem_cgroup_disabled()) 1020 return; 1021 1022 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); 1023 lru_size = mz->lru_size + lru; 1024 empty = list_empty(lruvec->lists + lru); 1025 1026 if (nr_pages < 0) 1027 *lru_size += nr_pages; 1028 1029 size = *lru_size; 1030 if (WARN_ONCE(size < 0 || empty != !size, 1031 "%s(%p, %d, %d): lru_size %ld but %sempty\n", 1032 __func__, lruvec, lru, nr_pages, size, empty ? "" : "not ")) { 1033 VM_BUG_ON(1); 1034 *lru_size = 0; 1035 } 1036 1037 if (nr_pages > 0) 1038 *lru_size += nr_pages; 1039 } 1040 1041 bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg) 1042 { 1043 struct mem_cgroup *task_memcg; 1044 struct task_struct *p; 1045 bool ret; 1046 1047 p = find_lock_task_mm(task); 1048 if (p) { 1049 task_memcg = get_mem_cgroup_from_mm(p->mm); 1050 task_unlock(p); 1051 } else { 1052 /* 1053 * All threads may have already detached their mm's, but the oom 1054 * killer still needs to detect if they have already been oom 1055 * killed to prevent needlessly killing additional tasks. 1056 */ 1057 rcu_read_lock(); 1058 task_memcg = mem_cgroup_from_task(task); 1059 css_get(&task_memcg->css); 1060 rcu_read_unlock(); 1061 } 1062 ret = mem_cgroup_is_descendant(task_memcg, memcg); 1063 css_put(&task_memcg->css); 1064 return ret; 1065 } 1066 1067 /** 1068 * mem_cgroup_margin - calculate chargeable space of a memory cgroup 1069 * @memcg: the memory cgroup 1070 * 1071 * Returns the maximum amount of memory @mem can be charged with, in 1072 * pages. 1073 */ 1074 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg) 1075 { 1076 unsigned long margin = 0; 1077 unsigned long count; 1078 unsigned long limit; 1079 1080 count = page_counter_read(&memcg->memory); 1081 limit = READ_ONCE(memcg->memory.limit); 1082 if (count < limit) 1083 margin = limit - count; 1084 1085 if (do_memsw_account()) { 1086 count = page_counter_read(&memcg->memsw); 1087 limit = READ_ONCE(memcg->memsw.limit); 1088 if (count <= limit) 1089 margin = min(margin, limit - count); 1090 else 1091 margin = 0; 1092 } 1093 1094 return margin; 1095 } 1096 1097 /* 1098 * A routine for checking "mem" is under move_account() or not. 1099 * 1100 * Checking a cgroup is mc.from or mc.to or under hierarchy of 1101 * moving cgroups. This is for waiting at high-memory pressure 1102 * caused by "move". 1103 */ 1104 static bool mem_cgroup_under_move(struct mem_cgroup *memcg) 1105 { 1106 struct mem_cgroup *from; 1107 struct mem_cgroup *to; 1108 bool ret = false; 1109 /* 1110 * Unlike task_move routines, we access mc.to, mc.from not under 1111 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead. 1112 */ 1113 spin_lock(&mc.lock); 1114 from = mc.from; 1115 to = mc.to; 1116 if (!from) 1117 goto unlock; 1118 1119 ret = mem_cgroup_is_descendant(from, memcg) || 1120 mem_cgroup_is_descendant(to, memcg); 1121 unlock: 1122 spin_unlock(&mc.lock); 1123 return ret; 1124 } 1125 1126 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg) 1127 { 1128 if (mc.moving_task && current != mc.moving_task) { 1129 if (mem_cgroup_under_move(memcg)) { 1130 DEFINE_WAIT(wait); 1131 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE); 1132 /* moving charge context might have finished. */ 1133 if (mc.moving_task) 1134 schedule(); 1135 finish_wait(&mc.waitq, &wait); 1136 return true; 1137 } 1138 } 1139 return false; 1140 } 1141 1142 #define K(x) ((x) << (PAGE_SHIFT-10)) 1143 /** 1144 * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller. 1145 * @memcg: The memory cgroup that went over limit 1146 * @p: Task that is going to be killed 1147 * 1148 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is 1149 * enabled 1150 */ 1151 void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p) 1152 { 1153 struct mem_cgroup *iter; 1154 unsigned int i; 1155 1156 rcu_read_lock(); 1157 1158 if (p) { 1159 pr_info("Task in "); 1160 pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id)); 1161 pr_cont(" killed as a result of limit of "); 1162 } else { 1163 pr_info("Memory limit reached of cgroup "); 1164 } 1165 1166 pr_cont_cgroup_path(memcg->css.cgroup); 1167 pr_cont("\n"); 1168 1169 rcu_read_unlock(); 1170 1171 pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n", 1172 K((u64)page_counter_read(&memcg->memory)), 1173 K((u64)memcg->memory.limit), memcg->memory.failcnt); 1174 pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n", 1175 K((u64)page_counter_read(&memcg->memsw)), 1176 K((u64)memcg->memsw.limit), memcg->memsw.failcnt); 1177 pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n", 1178 K((u64)page_counter_read(&memcg->kmem)), 1179 K((u64)memcg->kmem.limit), memcg->kmem.failcnt); 1180 1181 for_each_mem_cgroup_tree(iter, memcg) { 1182 pr_info("Memory cgroup stats for "); 1183 pr_cont_cgroup_path(iter->css.cgroup); 1184 pr_cont(":"); 1185 1186 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) { 1187 if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account) 1188 continue; 1189 pr_cont(" %s:%luKB", mem_cgroup_stat_names[i], 1190 K(mem_cgroup_read_stat(iter, i))); 1191 } 1192 1193 for (i = 0; i < NR_LRU_LISTS; i++) 1194 pr_cont(" %s:%luKB", mem_cgroup_lru_names[i], 1195 K(mem_cgroup_nr_lru_pages(iter, BIT(i)))); 1196 1197 pr_cont("\n"); 1198 } 1199 } 1200 1201 /* 1202 * This function returns the number of memcg under hierarchy tree. Returns 1203 * 1(self count) if no children. 1204 */ 1205 static int mem_cgroup_count_children(struct mem_cgroup *memcg) 1206 { 1207 int num = 0; 1208 struct mem_cgroup *iter; 1209 1210 for_each_mem_cgroup_tree(iter, memcg) 1211 num++; 1212 return num; 1213 } 1214 1215 /* 1216 * Return the memory (and swap, if configured) limit for a memcg. 1217 */ 1218 unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg) 1219 { 1220 unsigned long limit; 1221 1222 limit = memcg->memory.limit; 1223 if (mem_cgroup_swappiness(memcg)) { 1224 unsigned long memsw_limit; 1225 unsigned long swap_limit; 1226 1227 memsw_limit = memcg->memsw.limit; 1228 swap_limit = memcg->swap.limit; 1229 swap_limit = min(swap_limit, (unsigned long)total_swap_pages); 1230 limit = min(limit + swap_limit, memsw_limit); 1231 } 1232 return limit; 1233 } 1234 1235 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask, 1236 int order) 1237 { 1238 struct oom_control oc = { 1239 .zonelist = NULL, 1240 .nodemask = NULL, 1241 .memcg = memcg, 1242 .gfp_mask = gfp_mask, 1243 .order = order, 1244 }; 1245 bool ret; 1246 1247 mutex_lock(&oom_lock); 1248 ret = out_of_memory(&oc); 1249 mutex_unlock(&oom_lock); 1250 return ret; 1251 } 1252 1253 #if MAX_NUMNODES > 1 1254 1255 /** 1256 * test_mem_cgroup_node_reclaimable 1257 * @memcg: the target memcg 1258 * @nid: the node ID to be checked. 1259 * @noswap : specify true here if the user wants flle only information. 1260 * 1261 * This function returns whether the specified memcg contains any 1262 * reclaimable pages on a node. Returns true if there are any reclaimable 1263 * pages in the node. 1264 */ 1265 static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg, 1266 int nid, bool noswap) 1267 { 1268 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE)) 1269 return true; 1270 if (noswap || !total_swap_pages) 1271 return false; 1272 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON)) 1273 return true; 1274 return false; 1275 1276 } 1277 1278 /* 1279 * Always updating the nodemask is not very good - even if we have an empty 1280 * list or the wrong list here, we can start from some node and traverse all 1281 * nodes based on the zonelist. So update the list loosely once per 10 secs. 1282 * 1283 */ 1284 static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg) 1285 { 1286 int nid; 1287 /* 1288 * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET 1289 * pagein/pageout changes since the last update. 1290 */ 1291 if (!atomic_read(&memcg->numainfo_events)) 1292 return; 1293 if (atomic_inc_return(&memcg->numainfo_updating) > 1) 1294 return; 1295 1296 /* make a nodemask where this memcg uses memory from */ 1297 memcg->scan_nodes = node_states[N_MEMORY]; 1298 1299 for_each_node_mask(nid, node_states[N_MEMORY]) { 1300 1301 if (!test_mem_cgroup_node_reclaimable(memcg, nid, false)) 1302 node_clear(nid, memcg->scan_nodes); 1303 } 1304 1305 atomic_set(&memcg->numainfo_events, 0); 1306 atomic_set(&memcg->numainfo_updating, 0); 1307 } 1308 1309 /* 1310 * Selecting a node where we start reclaim from. Because what we need is just 1311 * reducing usage counter, start from anywhere is O,K. Considering 1312 * memory reclaim from current node, there are pros. and cons. 1313 * 1314 * Freeing memory from current node means freeing memory from a node which 1315 * we'll use or we've used. So, it may make LRU bad. And if several threads 1316 * hit limits, it will see a contention on a node. But freeing from remote 1317 * node means more costs for memory reclaim because of memory latency. 1318 * 1319 * Now, we use round-robin. Better algorithm is welcomed. 1320 */ 1321 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg) 1322 { 1323 int node; 1324 1325 mem_cgroup_may_update_nodemask(memcg); 1326 node = memcg->last_scanned_node; 1327 1328 node = next_node_in(node, memcg->scan_nodes); 1329 /* 1330 * mem_cgroup_may_update_nodemask might have seen no reclaimmable pages 1331 * last time it really checked all the LRUs due to rate limiting. 1332 * Fallback to the current node in that case for simplicity. 1333 */ 1334 if (unlikely(node == MAX_NUMNODES)) 1335 node = numa_node_id(); 1336 1337 memcg->last_scanned_node = node; 1338 return node; 1339 } 1340 #else 1341 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg) 1342 { 1343 return 0; 1344 } 1345 #endif 1346 1347 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg, 1348 pg_data_t *pgdat, 1349 gfp_t gfp_mask, 1350 unsigned long *total_scanned) 1351 { 1352 struct mem_cgroup *victim = NULL; 1353 int total = 0; 1354 int loop = 0; 1355 unsigned long excess; 1356 unsigned long nr_scanned; 1357 struct mem_cgroup_reclaim_cookie reclaim = { 1358 .pgdat = pgdat, 1359 .priority = 0, 1360 }; 1361 1362 excess = soft_limit_excess(root_memcg); 1363 1364 while (1) { 1365 victim = mem_cgroup_iter(root_memcg, victim, &reclaim); 1366 if (!victim) { 1367 loop++; 1368 if (loop >= 2) { 1369 /* 1370 * If we have not been able to reclaim 1371 * anything, it might because there are 1372 * no reclaimable pages under this hierarchy 1373 */ 1374 if (!total) 1375 break; 1376 /* 1377 * We want to do more targeted reclaim. 1378 * excess >> 2 is not to excessive so as to 1379 * reclaim too much, nor too less that we keep 1380 * coming back to reclaim from this cgroup 1381 */ 1382 if (total >= (excess >> 2) || 1383 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) 1384 break; 1385 } 1386 continue; 1387 } 1388 total += mem_cgroup_shrink_node(victim, gfp_mask, false, 1389 pgdat, &nr_scanned); 1390 *total_scanned += nr_scanned; 1391 if (!soft_limit_excess(root_memcg)) 1392 break; 1393 } 1394 mem_cgroup_iter_break(root_memcg, victim); 1395 return total; 1396 } 1397 1398 #ifdef CONFIG_LOCKDEP 1399 static struct lockdep_map memcg_oom_lock_dep_map = { 1400 .name = "memcg_oom_lock", 1401 }; 1402 #endif 1403 1404 static DEFINE_SPINLOCK(memcg_oom_lock); 1405 1406 /* 1407 * Check OOM-Killer is already running under our hierarchy. 1408 * If someone is running, return false. 1409 */ 1410 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg) 1411 { 1412 struct mem_cgroup *iter, *failed = NULL; 1413 1414 spin_lock(&memcg_oom_lock); 1415 1416 for_each_mem_cgroup_tree(iter, memcg) { 1417 if (iter->oom_lock) { 1418 /* 1419 * this subtree of our hierarchy is already locked 1420 * so we cannot give a lock. 1421 */ 1422 failed = iter; 1423 mem_cgroup_iter_break(memcg, iter); 1424 break; 1425 } else 1426 iter->oom_lock = true; 1427 } 1428 1429 if (failed) { 1430 /* 1431 * OK, we failed to lock the whole subtree so we have 1432 * to clean up what we set up to the failing subtree 1433 */ 1434 for_each_mem_cgroup_tree(iter, memcg) { 1435 if (iter == failed) { 1436 mem_cgroup_iter_break(memcg, iter); 1437 break; 1438 } 1439 iter->oom_lock = false; 1440 } 1441 } else 1442 mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_); 1443 1444 spin_unlock(&memcg_oom_lock); 1445 1446 return !failed; 1447 } 1448 1449 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg) 1450 { 1451 struct mem_cgroup *iter; 1452 1453 spin_lock(&memcg_oom_lock); 1454 mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_); 1455 for_each_mem_cgroup_tree(iter, memcg) 1456 iter->oom_lock = false; 1457 spin_unlock(&memcg_oom_lock); 1458 } 1459 1460 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg) 1461 { 1462 struct mem_cgroup *iter; 1463 1464 spin_lock(&memcg_oom_lock); 1465 for_each_mem_cgroup_tree(iter, memcg) 1466 iter->under_oom++; 1467 spin_unlock(&memcg_oom_lock); 1468 } 1469 1470 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg) 1471 { 1472 struct mem_cgroup *iter; 1473 1474 /* 1475 * When a new child is created while the hierarchy is under oom, 1476 * mem_cgroup_oom_lock() may not be called. Watch for underflow. 1477 */ 1478 spin_lock(&memcg_oom_lock); 1479 for_each_mem_cgroup_tree(iter, memcg) 1480 if (iter->under_oom > 0) 1481 iter->under_oom--; 1482 spin_unlock(&memcg_oom_lock); 1483 } 1484 1485 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq); 1486 1487 struct oom_wait_info { 1488 struct mem_cgroup *memcg; 1489 wait_queue_t wait; 1490 }; 1491 1492 static int memcg_oom_wake_function(wait_queue_t *wait, 1493 unsigned mode, int sync, void *arg) 1494 { 1495 struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg; 1496 struct mem_cgroup *oom_wait_memcg; 1497 struct oom_wait_info *oom_wait_info; 1498 1499 oom_wait_info = container_of(wait, struct oom_wait_info, wait); 1500 oom_wait_memcg = oom_wait_info->memcg; 1501 1502 if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) && 1503 !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg)) 1504 return 0; 1505 return autoremove_wake_function(wait, mode, sync, arg); 1506 } 1507 1508 static void memcg_oom_recover(struct mem_cgroup *memcg) 1509 { 1510 /* 1511 * For the following lockless ->under_oom test, the only required 1512 * guarantee is that it must see the state asserted by an OOM when 1513 * this function is called as a result of userland actions 1514 * triggered by the notification of the OOM. This is trivially 1515 * achieved by invoking mem_cgroup_mark_under_oom() before 1516 * triggering notification. 1517 */ 1518 if (memcg && memcg->under_oom) 1519 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg); 1520 } 1521 1522 static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order) 1523 { 1524 if (!current->memcg_may_oom) 1525 return; 1526 /* 1527 * We are in the middle of the charge context here, so we 1528 * don't want to block when potentially sitting on a callstack 1529 * that holds all kinds of filesystem and mm locks. 1530 * 1531 * Also, the caller may handle a failed allocation gracefully 1532 * (like optional page cache readahead) and so an OOM killer 1533 * invocation might not even be necessary. 1534 * 1535 * That's why we don't do anything here except remember the 1536 * OOM context and then deal with it at the end of the page 1537 * fault when the stack is unwound, the locks are released, 1538 * and when we know whether the fault was overall successful. 1539 */ 1540 css_get(&memcg->css); 1541 current->memcg_in_oom = memcg; 1542 current->memcg_oom_gfp_mask = mask; 1543 current->memcg_oom_order = order; 1544 } 1545 1546 /** 1547 * mem_cgroup_oom_synchronize - complete memcg OOM handling 1548 * @handle: actually kill/wait or just clean up the OOM state 1549 * 1550 * This has to be called at the end of a page fault if the memcg OOM 1551 * handler was enabled. 1552 * 1553 * Memcg supports userspace OOM handling where failed allocations must 1554 * sleep on a waitqueue until the userspace task resolves the 1555 * situation. Sleeping directly in the charge context with all kinds 1556 * of locks held is not a good idea, instead we remember an OOM state 1557 * in the task and mem_cgroup_oom_synchronize() has to be called at 1558 * the end of the page fault to complete the OOM handling. 1559 * 1560 * Returns %true if an ongoing memcg OOM situation was detected and 1561 * completed, %false otherwise. 1562 */ 1563 bool mem_cgroup_oom_synchronize(bool handle) 1564 { 1565 struct mem_cgroup *memcg = current->memcg_in_oom; 1566 struct oom_wait_info owait; 1567 bool locked; 1568 1569 /* OOM is global, do not handle */ 1570 if (!memcg) 1571 return false; 1572 1573 if (!handle) 1574 goto cleanup; 1575 1576 owait.memcg = memcg; 1577 owait.wait.flags = 0; 1578 owait.wait.func = memcg_oom_wake_function; 1579 owait.wait.private = current; 1580 INIT_LIST_HEAD(&owait.wait.task_list); 1581 1582 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE); 1583 mem_cgroup_mark_under_oom(memcg); 1584 1585 locked = mem_cgroup_oom_trylock(memcg); 1586 1587 if (locked) 1588 mem_cgroup_oom_notify(memcg); 1589 1590 if (locked && !memcg->oom_kill_disable) { 1591 mem_cgroup_unmark_under_oom(memcg); 1592 finish_wait(&memcg_oom_waitq, &owait.wait); 1593 mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask, 1594 current->memcg_oom_order); 1595 } else { 1596 schedule(); 1597 mem_cgroup_unmark_under_oom(memcg); 1598 finish_wait(&memcg_oom_waitq, &owait.wait); 1599 } 1600 1601 if (locked) { 1602 mem_cgroup_oom_unlock(memcg); 1603 /* 1604 * There is no guarantee that an OOM-lock contender 1605 * sees the wakeups triggered by the OOM kill 1606 * uncharges. Wake any sleepers explicitely. 1607 */ 1608 memcg_oom_recover(memcg); 1609 } 1610 cleanup: 1611 current->memcg_in_oom = NULL; 1612 css_put(&memcg->css); 1613 return true; 1614 } 1615 1616 /** 1617 * lock_page_memcg - lock a page->mem_cgroup binding 1618 * @page: the page 1619 * 1620 * This function protects unlocked LRU pages from being moved to 1621 * another cgroup and stabilizes their page->mem_cgroup binding. 1622 */ 1623 void lock_page_memcg(struct page *page) 1624 { 1625 struct mem_cgroup *memcg; 1626 unsigned long flags; 1627 1628 /* 1629 * The RCU lock is held throughout the transaction. The fast 1630 * path can get away without acquiring the memcg->move_lock 1631 * because page moving starts with an RCU grace period. 1632 */ 1633 rcu_read_lock(); 1634 1635 if (mem_cgroup_disabled()) 1636 return; 1637 again: 1638 memcg = page->mem_cgroup; 1639 if (unlikely(!memcg)) 1640 return; 1641 1642 if (atomic_read(&memcg->moving_account) <= 0) 1643 return; 1644 1645 spin_lock_irqsave(&memcg->move_lock, flags); 1646 if (memcg != page->mem_cgroup) { 1647 spin_unlock_irqrestore(&memcg->move_lock, flags); 1648 goto again; 1649 } 1650 1651 /* 1652 * When charge migration first begins, we can have locked and 1653 * unlocked page stat updates happening concurrently. Track 1654 * the task who has the lock for unlock_page_memcg(). 1655 */ 1656 memcg->move_lock_task = current; 1657 memcg->move_lock_flags = flags; 1658 1659 return; 1660 } 1661 EXPORT_SYMBOL(lock_page_memcg); 1662 1663 /** 1664 * unlock_page_memcg - unlock a page->mem_cgroup binding 1665 * @page: the page 1666 */ 1667 void unlock_page_memcg(struct page *page) 1668 { 1669 struct mem_cgroup *memcg = page->mem_cgroup; 1670 1671 if (memcg && memcg->move_lock_task == current) { 1672 unsigned long flags = memcg->move_lock_flags; 1673 1674 memcg->move_lock_task = NULL; 1675 memcg->move_lock_flags = 0; 1676 1677 spin_unlock_irqrestore(&memcg->move_lock, flags); 1678 } 1679 1680 rcu_read_unlock(); 1681 } 1682 EXPORT_SYMBOL(unlock_page_memcg); 1683 1684 /* 1685 * size of first charge trial. "32" comes from vmscan.c's magic value. 1686 * TODO: maybe necessary to use big numbers in big irons. 1687 */ 1688 #define CHARGE_BATCH 32U 1689 struct memcg_stock_pcp { 1690 struct mem_cgroup *cached; /* this never be root cgroup */ 1691 unsigned int nr_pages; 1692 struct work_struct work; 1693 unsigned long flags; 1694 #define FLUSHING_CACHED_CHARGE 0 1695 }; 1696 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock); 1697 static DEFINE_MUTEX(percpu_charge_mutex); 1698 1699 /** 1700 * consume_stock: Try to consume stocked charge on this cpu. 1701 * @memcg: memcg to consume from. 1702 * @nr_pages: how many pages to charge. 1703 * 1704 * The charges will only happen if @memcg matches the current cpu's memcg 1705 * stock, and at least @nr_pages are available in that stock. Failure to 1706 * service an allocation will refill the stock. 1707 * 1708 * returns true if successful, false otherwise. 1709 */ 1710 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) 1711 { 1712 struct memcg_stock_pcp *stock; 1713 unsigned long flags; 1714 bool ret = false; 1715 1716 if (nr_pages > CHARGE_BATCH) 1717 return ret; 1718 1719 local_irq_save(flags); 1720 1721 stock = this_cpu_ptr(&memcg_stock); 1722 if (memcg == stock->cached && stock->nr_pages >= nr_pages) { 1723 stock->nr_pages -= nr_pages; 1724 ret = true; 1725 } 1726 1727 local_irq_restore(flags); 1728 1729 return ret; 1730 } 1731 1732 /* 1733 * Returns stocks cached in percpu and reset cached information. 1734 */ 1735 static void drain_stock(struct memcg_stock_pcp *stock) 1736 { 1737 struct mem_cgroup *old = stock->cached; 1738 1739 if (stock->nr_pages) { 1740 page_counter_uncharge(&old->memory, stock->nr_pages); 1741 if (do_memsw_account()) 1742 page_counter_uncharge(&old->memsw, stock->nr_pages); 1743 css_put_many(&old->css, stock->nr_pages); 1744 stock->nr_pages = 0; 1745 } 1746 stock->cached = NULL; 1747 } 1748 1749 static void drain_local_stock(struct work_struct *dummy) 1750 { 1751 struct memcg_stock_pcp *stock; 1752 unsigned long flags; 1753 1754 local_irq_save(flags); 1755 1756 stock = this_cpu_ptr(&memcg_stock); 1757 drain_stock(stock); 1758 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags); 1759 1760 local_irq_restore(flags); 1761 } 1762 1763 /* 1764 * Cache charges(val) to local per_cpu area. 1765 * This will be consumed by consume_stock() function, later. 1766 */ 1767 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) 1768 { 1769 struct memcg_stock_pcp *stock; 1770 unsigned long flags; 1771 1772 local_irq_save(flags); 1773 1774 stock = this_cpu_ptr(&memcg_stock); 1775 if (stock->cached != memcg) { /* reset if necessary */ 1776 drain_stock(stock); 1777 stock->cached = memcg; 1778 } 1779 stock->nr_pages += nr_pages; 1780 1781 local_irq_restore(flags); 1782 } 1783 1784 /* 1785 * Drains all per-CPU charge caches for given root_memcg resp. subtree 1786 * of the hierarchy under it. 1787 */ 1788 static void drain_all_stock(struct mem_cgroup *root_memcg) 1789 { 1790 int cpu, curcpu; 1791 1792 /* If someone's already draining, avoid adding running more workers. */ 1793 if (!mutex_trylock(&percpu_charge_mutex)) 1794 return; 1795 /* Notify other cpus that system-wide "drain" is running */ 1796 get_online_cpus(); 1797 curcpu = get_cpu(); 1798 for_each_online_cpu(cpu) { 1799 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); 1800 struct mem_cgroup *memcg; 1801 1802 memcg = stock->cached; 1803 if (!memcg || !stock->nr_pages) 1804 continue; 1805 if (!mem_cgroup_is_descendant(memcg, root_memcg)) 1806 continue; 1807 if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) { 1808 if (cpu == curcpu) 1809 drain_local_stock(&stock->work); 1810 else 1811 schedule_work_on(cpu, &stock->work); 1812 } 1813 } 1814 put_cpu(); 1815 put_online_cpus(); 1816 mutex_unlock(&percpu_charge_mutex); 1817 } 1818 1819 static int memcg_hotplug_cpu_dead(unsigned int cpu) 1820 { 1821 struct memcg_stock_pcp *stock; 1822 1823 stock = &per_cpu(memcg_stock, cpu); 1824 drain_stock(stock); 1825 return 0; 1826 } 1827 1828 static void reclaim_high(struct mem_cgroup *memcg, 1829 unsigned int nr_pages, 1830 gfp_t gfp_mask) 1831 { 1832 do { 1833 if (page_counter_read(&memcg->memory) <= memcg->high) 1834 continue; 1835 mem_cgroup_events(memcg, MEMCG_HIGH, 1); 1836 try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true); 1837 } while ((memcg = parent_mem_cgroup(memcg))); 1838 } 1839 1840 static void high_work_func(struct work_struct *work) 1841 { 1842 struct mem_cgroup *memcg; 1843 1844 memcg = container_of(work, struct mem_cgroup, high_work); 1845 reclaim_high(memcg, CHARGE_BATCH, GFP_KERNEL); 1846 } 1847 1848 /* 1849 * Scheduled by try_charge() to be executed from the userland return path 1850 * and reclaims memory over the high limit. 1851 */ 1852 void mem_cgroup_handle_over_high(void) 1853 { 1854 unsigned int nr_pages = current->memcg_nr_pages_over_high; 1855 struct mem_cgroup *memcg; 1856 1857 if (likely(!nr_pages)) 1858 return; 1859 1860 memcg = get_mem_cgroup_from_mm(current->mm); 1861 reclaim_high(memcg, nr_pages, GFP_KERNEL); 1862 css_put(&memcg->css); 1863 current->memcg_nr_pages_over_high = 0; 1864 } 1865 1866 static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, 1867 unsigned int nr_pages) 1868 { 1869 unsigned int batch = max(CHARGE_BATCH, nr_pages); 1870 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES; 1871 struct mem_cgroup *mem_over_limit; 1872 struct page_counter *counter; 1873 unsigned long nr_reclaimed; 1874 bool may_swap = true; 1875 bool drained = false; 1876 1877 if (mem_cgroup_is_root(memcg)) 1878 return 0; 1879 retry: 1880 if (consume_stock(memcg, nr_pages)) 1881 return 0; 1882 1883 if (!do_memsw_account() || 1884 page_counter_try_charge(&memcg->memsw, batch, &counter)) { 1885 if (page_counter_try_charge(&memcg->memory, batch, &counter)) 1886 goto done_restock; 1887 if (do_memsw_account()) 1888 page_counter_uncharge(&memcg->memsw, batch); 1889 mem_over_limit = mem_cgroup_from_counter(counter, memory); 1890 } else { 1891 mem_over_limit = mem_cgroup_from_counter(counter, memsw); 1892 may_swap = false; 1893 } 1894 1895 if (batch > nr_pages) { 1896 batch = nr_pages; 1897 goto retry; 1898 } 1899 1900 /* 1901 * Unlike in global OOM situations, memcg is not in a physical 1902 * memory shortage. Allow dying and OOM-killed tasks to 1903 * bypass the last charges so that they can exit quickly and 1904 * free their memory. 1905 */ 1906 if (unlikely(test_thread_flag(TIF_MEMDIE) || 1907 fatal_signal_pending(current) || 1908 current->flags & PF_EXITING)) 1909 goto force; 1910 1911 /* 1912 * Prevent unbounded recursion when reclaim operations need to 1913 * allocate memory. This might exceed the limits temporarily, 1914 * but we prefer facilitating memory reclaim and getting back 1915 * under the limit over triggering OOM kills in these cases. 1916 */ 1917 if (unlikely(current->flags & PF_MEMALLOC)) 1918 goto force; 1919 1920 if (unlikely(task_in_memcg_oom(current))) 1921 goto nomem; 1922 1923 if (!gfpflags_allow_blocking(gfp_mask)) 1924 goto nomem; 1925 1926 mem_cgroup_events(mem_over_limit, MEMCG_MAX, 1); 1927 1928 nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages, 1929 gfp_mask, may_swap); 1930 1931 if (mem_cgroup_margin(mem_over_limit) >= nr_pages) 1932 goto retry; 1933 1934 if (!drained) { 1935 drain_all_stock(mem_over_limit); 1936 drained = true; 1937 goto retry; 1938 } 1939 1940 if (gfp_mask & __GFP_NORETRY) 1941 goto nomem; 1942 /* 1943 * Even though the limit is exceeded at this point, reclaim 1944 * may have been able to free some pages. Retry the charge 1945 * before killing the task. 1946 * 1947 * Only for regular pages, though: huge pages are rather 1948 * unlikely to succeed so close to the limit, and we fall back 1949 * to regular pages anyway in case of failure. 1950 */ 1951 if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER)) 1952 goto retry; 1953 /* 1954 * At task move, charge accounts can be doubly counted. So, it's 1955 * better to wait until the end of task_move if something is going on. 1956 */ 1957 if (mem_cgroup_wait_acct_move(mem_over_limit)) 1958 goto retry; 1959 1960 if (nr_retries--) 1961 goto retry; 1962 1963 if (gfp_mask & __GFP_NOFAIL) 1964 goto force; 1965 1966 if (fatal_signal_pending(current)) 1967 goto force; 1968 1969 mem_cgroup_events(mem_over_limit, MEMCG_OOM, 1); 1970 1971 mem_cgroup_oom(mem_over_limit, gfp_mask, 1972 get_order(nr_pages * PAGE_SIZE)); 1973 nomem: 1974 if (!(gfp_mask & __GFP_NOFAIL)) 1975 return -ENOMEM; 1976 force: 1977 /* 1978 * The allocation either can't fail or will lead to more memory 1979 * being freed very soon. Allow memory usage go over the limit 1980 * temporarily by force charging it. 1981 */ 1982 page_counter_charge(&memcg->memory, nr_pages); 1983 if (do_memsw_account()) 1984 page_counter_charge(&memcg->memsw, nr_pages); 1985 css_get_many(&memcg->css, nr_pages); 1986 1987 return 0; 1988 1989 done_restock: 1990 css_get_many(&memcg->css, batch); 1991 if (batch > nr_pages) 1992 refill_stock(memcg, batch - nr_pages); 1993 1994 /* 1995 * If the hierarchy is above the normal consumption range, schedule 1996 * reclaim on returning to userland. We can perform reclaim here 1997 * if __GFP_RECLAIM but let's always punt for simplicity and so that 1998 * GFP_KERNEL can consistently be used during reclaim. @memcg is 1999 * not recorded as it most likely matches current's and won't 2000 * change in the meantime. As high limit is checked again before 2001 * reclaim, the cost of mismatch is negligible. 2002 */ 2003 do { 2004 if (page_counter_read(&memcg->memory) > memcg->high) { 2005 /* Don't bother a random interrupted task */ 2006 if (in_interrupt()) { 2007 schedule_work(&memcg->high_work); 2008 break; 2009 } 2010 current->memcg_nr_pages_over_high += batch; 2011 set_notify_resume(current); 2012 break; 2013 } 2014 } while ((memcg = parent_mem_cgroup(memcg))); 2015 2016 return 0; 2017 } 2018 2019 static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages) 2020 { 2021 if (mem_cgroup_is_root(memcg)) 2022 return; 2023 2024 page_counter_uncharge(&memcg->memory, nr_pages); 2025 if (do_memsw_account()) 2026 page_counter_uncharge(&memcg->memsw, nr_pages); 2027 2028 css_put_many(&memcg->css, nr_pages); 2029 } 2030 2031 static void lock_page_lru(struct page *page, int *isolated) 2032 { 2033 struct zone *zone = page_zone(page); 2034 2035 spin_lock_irq(zone_lru_lock(zone)); 2036 if (PageLRU(page)) { 2037 struct lruvec *lruvec; 2038 2039 lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat); 2040 ClearPageLRU(page); 2041 del_page_from_lru_list(page, lruvec, page_lru(page)); 2042 *isolated = 1; 2043 } else 2044 *isolated = 0; 2045 } 2046 2047 static void unlock_page_lru(struct page *page, int isolated) 2048 { 2049 struct zone *zone = page_zone(page); 2050 2051 if (isolated) { 2052 struct lruvec *lruvec; 2053 2054 lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat); 2055 VM_BUG_ON_PAGE(PageLRU(page), page); 2056 SetPageLRU(page); 2057 add_page_to_lru_list(page, lruvec, page_lru(page)); 2058 } 2059 spin_unlock_irq(zone_lru_lock(zone)); 2060 } 2061 2062 static void commit_charge(struct page *page, struct mem_cgroup *memcg, 2063 bool lrucare) 2064 { 2065 int isolated; 2066 2067 VM_BUG_ON_PAGE(page->mem_cgroup, page); 2068 2069 /* 2070 * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page 2071 * may already be on some other mem_cgroup's LRU. Take care of it. 2072 */ 2073 if (lrucare) 2074 lock_page_lru(page, &isolated); 2075 2076 /* 2077 * Nobody should be changing or seriously looking at 2078 * page->mem_cgroup at this point: 2079 * 2080 * - the page is uncharged 2081 * 2082 * - the page is off-LRU 2083 * 2084 * - an anonymous fault has exclusive page access, except for 2085 * a locked page table 2086 * 2087 * - a page cache insertion, a swapin fault, or a migration 2088 * have the page locked 2089 */ 2090 page->mem_cgroup = memcg; 2091 2092 if (lrucare) 2093 unlock_page_lru(page, isolated); 2094 } 2095 2096 #ifndef CONFIG_SLOB 2097 static int memcg_alloc_cache_id(void) 2098 { 2099 int id, size; 2100 int err; 2101 2102 id = ida_simple_get(&memcg_cache_ida, 2103 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL); 2104 if (id < 0) 2105 return id; 2106 2107 if (id < memcg_nr_cache_ids) 2108 return id; 2109 2110 /* 2111 * There's no space for the new id in memcg_caches arrays, 2112 * so we have to grow them. 2113 */ 2114 down_write(&memcg_cache_ids_sem); 2115 2116 size = 2 * (id + 1); 2117 if (size < MEMCG_CACHES_MIN_SIZE) 2118 size = MEMCG_CACHES_MIN_SIZE; 2119 else if (size > MEMCG_CACHES_MAX_SIZE) 2120 size = MEMCG_CACHES_MAX_SIZE; 2121 2122 err = memcg_update_all_caches(size); 2123 if (!err) 2124 err = memcg_update_all_list_lrus(size); 2125 if (!err) 2126 memcg_nr_cache_ids = size; 2127 2128 up_write(&memcg_cache_ids_sem); 2129 2130 if (err) { 2131 ida_simple_remove(&memcg_cache_ida, id); 2132 return err; 2133 } 2134 return id; 2135 } 2136 2137 static void memcg_free_cache_id(int id) 2138 { 2139 ida_simple_remove(&memcg_cache_ida, id); 2140 } 2141 2142 struct memcg_kmem_cache_create_work { 2143 struct mem_cgroup *memcg; 2144 struct kmem_cache *cachep; 2145 struct work_struct work; 2146 }; 2147 2148 static struct workqueue_struct *memcg_kmem_cache_create_wq; 2149 2150 static void memcg_kmem_cache_create_func(struct work_struct *w) 2151 { 2152 struct memcg_kmem_cache_create_work *cw = 2153 container_of(w, struct memcg_kmem_cache_create_work, work); 2154 struct mem_cgroup *memcg = cw->memcg; 2155 struct kmem_cache *cachep = cw->cachep; 2156 2157 memcg_create_kmem_cache(memcg, cachep); 2158 2159 css_put(&memcg->css); 2160 kfree(cw); 2161 } 2162 2163 /* 2164 * Enqueue the creation of a per-memcg kmem_cache. 2165 */ 2166 static void __memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg, 2167 struct kmem_cache *cachep) 2168 { 2169 struct memcg_kmem_cache_create_work *cw; 2170 2171 cw = kmalloc(sizeof(*cw), GFP_NOWAIT); 2172 if (!cw) 2173 return; 2174 2175 css_get(&memcg->css); 2176 2177 cw->memcg = memcg; 2178 cw->cachep = cachep; 2179 INIT_WORK(&cw->work, memcg_kmem_cache_create_func); 2180 2181 queue_work(memcg_kmem_cache_create_wq, &cw->work); 2182 } 2183 2184 static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg, 2185 struct kmem_cache *cachep) 2186 { 2187 /* 2188 * We need to stop accounting when we kmalloc, because if the 2189 * corresponding kmalloc cache is not yet created, the first allocation 2190 * in __memcg_schedule_kmem_cache_create will recurse. 2191 * 2192 * However, it is better to enclose the whole function. Depending on 2193 * the debugging options enabled, INIT_WORK(), for instance, can 2194 * trigger an allocation. This too, will make us recurse. Because at 2195 * this point we can't allow ourselves back into memcg_kmem_get_cache, 2196 * the safest choice is to do it like this, wrapping the whole function. 2197 */ 2198 current->memcg_kmem_skip_account = 1; 2199 __memcg_schedule_kmem_cache_create(memcg, cachep); 2200 current->memcg_kmem_skip_account = 0; 2201 } 2202 2203 static inline bool memcg_kmem_bypass(void) 2204 { 2205 if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD)) 2206 return true; 2207 return false; 2208 } 2209 2210 /** 2211 * memcg_kmem_get_cache: select the correct per-memcg cache for allocation 2212 * @cachep: the original global kmem cache 2213 * 2214 * Return the kmem_cache we're supposed to use for a slab allocation. 2215 * We try to use the current memcg's version of the cache. 2216 * 2217 * If the cache does not exist yet, if we are the first user of it, we 2218 * create it asynchronously in a workqueue and let the current allocation 2219 * go through with the original cache. 2220 * 2221 * This function takes a reference to the cache it returns to assure it 2222 * won't get destroyed while we are working with it. Once the caller is 2223 * done with it, memcg_kmem_put_cache() must be called to release the 2224 * reference. 2225 */ 2226 struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep) 2227 { 2228 struct mem_cgroup *memcg; 2229 struct kmem_cache *memcg_cachep; 2230 int kmemcg_id; 2231 2232 VM_BUG_ON(!is_root_cache(cachep)); 2233 2234 if (memcg_kmem_bypass()) 2235 return cachep; 2236 2237 if (current->memcg_kmem_skip_account) 2238 return cachep; 2239 2240 memcg = get_mem_cgroup_from_mm(current->mm); 2241 kmemcg_id = READ_ONCE(memcg->kmemcg_id); 2242 if (kmemcg_id < 0) 2243 goto out; 2244 2245 memcg_cachep = cache_from_memcg_idx(cachep, kmemcg_id); 2246 if (likely(memcg_cachep)) 2247 return memcg_cachep; 2248 2249 /* 2250 * If we are in a safe context (can wait, and not in interrupt 2251 * context), we could be be predictable and return right away. 2252 * This would guarantee that the allocation being performed 2253 * already belongs in the new cache. 2254 * 2255 * However, there are some clashes that can arrive from locking. 2256 * For instance, because we acquire the slab_mutex while doing 2257 * memcg_create_kmem_cache, this means no further allocation 2258 * could happen with the slab_mutex held. So it's better to 2259 * defer everything. 2260 */ 2261 memcg_schedule_kmem_cache_create(memcg, cachep); 2262 out: 2263 css_put(&memcg->css); 2264 return cachep; 2265 } 2266 2267 /** 2268 * memcg_kmem_put_cache: drop reference taken by memcg_kmem_get_cache 2269 * @cachep: the cache returned by memcg_kmem_get_cache 2270 */ 2271 void memcg_kmem_put_cache(struct kmem_cache *cachep) 2272 { 2273 if (!is_root_cache(cachep)) 2274 css_put(&cachep->memcg_params.memcg->css); 2275 } 2276 2277 /** 2278 * memcg_kmem_charge: charge a kmem page 2279 * @page: page to charge 2280 * @gfp: reclaim mode 2281 * @order: allocation order 2282 * @memcg: memory cgroup to charge 2283 * 2284 * Returns 0 on success, an error code on failure. 2285 */ 2286 int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order, 2287 struct mem_cgroup *memcg) 2288 { 2289 unsigned int nr_pages = 1 << order; 2290 struct page_counter *counter; 2291 int ret; 2292 2293 ret = try_charge(memcg, gfp, nr_pages); 2294 if (ret) 2295 return ret; 2296 2297 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && 2298 !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) { 2299 cancel_charge(memcg, nr_pages); 2300 return -ENOMEM; 2301 } 2302 2303 page->mem_cgroup = memcg; 2304 2305 return 0; 2306 } 2307 2308 /** 2309 * memcg_kmem_charge: charge a kmem page to the current memory cgroup 2310 * @page: page to charge 2311 * @gfp: reclaim mode 2312 * @order: allocation order 2313 * 2314 * Returns 0 on success, an error code on failure. 2315 */ 2316 int memcg_kmem_charge(struct page *page, gfp_t gfp, int order) 2317 { 2318 struct mem_cgroup *memcg; 2319 int ret = 0; 2320 2321 if (memcg_kmem_bypass()) 2322 return 0; 2323 2324 memcg = get_mem_cgroup_from_mm(current->mm); 2325 if (!mem_cgroup_is_root(memcg)) { 2326 ret = memcg_kmem_charge_memcg(page, gfp, order, memcg); 2327 if (!ret) 2328 __SetPageKmemcg(page); 2329 } 2330 css_put(&memcg->css); 2331 return ret; 2332 } 2333 /** 2334 * memcg_kmem_uncharge: uncharge a kmem page 2335 * @page: page to uncharge 2336 * @order: allocation order 2337 */ 2338 void memcg_kmem_uncharge(struct page *page, int order) 2339 { 2340 struct mem_cgroup *memcg = page->mem_cgroup; 2341 unsigned int nr_pages = 1 << order; 2342 2343 if (!memcg) 2344 return; 2345 2346 VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page); 2347 2348 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) 2349 page_counter_uncharge(&memcg->kmem, nr_pages); 2350 2351 page_counter_uncharge(&memcg->memory, nr_pages); 2352 if (do_memsw_account()) 2353 page_counter_uncharge(&memcg->memsw, nr_pages); 2354 2355 page->mem_cgroup = NULL; 2356 2357 /* slab pages do not have PageKmemcg flag set */ 2358 if (PageKmemcg(page)) 2359 __ClearPageKmemcg(page); 2360 2361 css_put_many(&memcg->css, nr_pages); 2362 } 2363 #endif /* !CONFIG_SLOB */ 2364 2365 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2366 2367 /* 2368 * Because tail pages are not marked as "used", set it. We're under 2369 * zone_lru_lock and migration entries setup in all page mappings. 2370 */ 2371 void mem_cgroup_split_huge_fixup(struct page *head) 2372 { 2373 int i; 2374 2375 if (mem_cgroup_disabled()) 2376 return; 2377 2378 for (i = 1; i < HPAGE_PMD_NR; i++) 2379 head[i].mem_cgroup = head->mem_cgroup; 2380 2381 __this_cpu_sub(head->mem_cgroup->stat->count[MEM_CGROUP_STAT_RSS_HUGE], 2382 HPAGE_PMD_NR); 2383 } 2384 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 2385 2386 #ifdef CONFIG_MEMCG_SWAP 2387 static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg, 2388 bool charge) 2389 { 2390 int val = (charge) ? 1 : -1; 2391 this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAP], val); 2392 } 2393 2394 /** 2395 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record. 2396 * @entry: swap entry to be moved 2397 * @from: mem_cgroup which the entry is moved from 2398 * @to: mem_cgroup which the entry is moved to 2399 * 2400 * It succeeds only when the swap_cgroup's record for this entry is the same 2401 * as the mem_cgroup's id of @from. 2402 * 2403 * Returns 0 on success, -EINVAL on failure. 2404 * 2405 * The caller must have charged to @to, IOW, called page_counter_charge() about 2406 * both res and memsw, and called css_get(). 2407 */ 2408 static int mem_cgroup_move_swap_account(swp_entry_t entry, 2409 struct mem_cgroup *from, struct mem_cgroup *to) 2410 { 2411 unsigned short old_id, new_id; 2412 2413 old_id = mem_cgroup_id(from); 2414 new_id = mem_cgroup_id(to); 2415 2416 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) { 2417 mem_cgroup_swap_statistics(from, false); 2418 mem_cgroup_swap_statistics(to, true); 2419 return 0; 2420 } 2421 return -EINVAL; 2422 } 2423 #else 2424 static inline int mem_cgroup_move_swap_account(swp_entry_t entry, 2425 struct mem_cgroup *from, struct mem_cgroup *to) 2426 { 2427 return -EINVAL; 2428 } 2429 #endif 2430 2431 static DEFINE_MUTEX(memcg_limit_mutex); 2432 2433 static int mem_cgroup_resize_limit(struct mem_cgroup *memcg, 2434 unsigned long limit) 2435 { 2436 unsigned long curusage; 2437 unsigned long oldusage; 2438 bool enlarge = false; 2439 int retry_count; 2440 int ret; 2441 2442 /* 2443 * For keeping hierarchical_reclaim simple, how long we should retry 2444 * is depends on callers. We set our retry-count to be function 2445 * of # of children which we should visit in this loop. 2446 */ 2447 retry_count = MEM_CGROUP_RECLAIM_RETRIES * 2448 mem_cgroup_count_children(memcg); 2449 2450 oldusage = page_counter_read(&memcg->memory); 2451 2452 do { 2453 if (signal_pending(current)) { 2454 ret = -EINTR; 2455 break; 2456 } 2457 2458 mutex_lock(&memcg_limit_mutex); 2459 if (limit > memcg->memsw.limit) { 2460 mutex_unlock(&memcg_limit_mutex); 2461 ret = -EINVAL; 2462 break; 2463 } 2464 if (limit > memcg->memory.limit) 2465 enlarge = true; 2466 ret = page_counter_limit(&memcg->memory, limit); 2467 mutex_unlock(&memcg_limit_mutex); 2468 2469 if (!ret) 2470 break; 2471 2472 try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, true); 2473 2474 curusage = page_counter_read(&memcg->memory); 2475 /* Usage is reduced ? */ 2476 if (curusage >= oldusage) 2477 retry_count--; 2478 else 2479 oldusage = curusage; 2480 } while (retry_count); 2481 2482 if (!ret && enlarge) 2483 memcg_oom_recover(memcg); 2484 2485 return ret; 2486 } 2487 2488 static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg, 2489 unsigned long limit) 2490 { 2491 unsigned long curusage; 2492 unsigned long oldusage; 2493 bool enlarge = false; 2494 int retry_count; 2495 int ret; 2496 2497 /* see mem_cgroup_resize_res_limit */ 2498 retry_count = MEM_CGROUP_RECLAIM_RETRIES * 2499 mem_cgroup_count_children(memcg); 2500 2501 oldusage = page_counter_read(&memcg->memsw); 2502 2503 do { 2504 if (signal_pending(current)) { 2505 ret = -EINTR; 2506 break; 2507 } 2508 2509 mutex_lock(&memcg_limit_mutex); 2510 if (limit < memcg->memory.limit) { 2511 mutex_unlock(&memcg_limit_mutex); 2512 ret = -EINVAL; 2513 break; 2514 } 2515 if (limit > memcg->memsw.limit) 2516 enlarge = true; 2517 ret = page_counter_limit(&memcg->memsw, limit); 2518 mutex_unlock(&memcg_limit_mutex); 2519 2520 if (!ret) 2521 break; 2522 2523 try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, false); 2524 2525 curusage = page_counter_read(&memcg->memsw); 2526 /* Usage is reduced ? */ 2527 if (curusage >= oldusage) 2528 retry_count--; 2529 else 2530 oldusage = curusage; 2531 } while (retry_count); 2532 2533 if (!ret && enlarge) 2534 memcg_oom_recover(memcg); 2535 2536 return ret; 2537 } 2538 2539 unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order, 2540 gfp_t gfp_mask, 2541 unsigned long *total_scanned) 2542 { 2543 unsigned long nr_reclaimed = 0; 2544 struct mem_cgroup_per_node *mz, *next_mz = NULL; 2545 unsigned long reclaimed; 2546 int loop = 0; 2547 struct mem_cgroup_tree_per_node *mctz; 2548 unsigned long excess; 2549 unsigned long nr_scanned; 2550 2551 if (order > 0) 2552 return 0; 2553 2554 mctz = soft_limit_tree_node(pgdat->node_id); 2555 2556 /* 2557 * Do not even bother to check the largest node if the root 2558 * is empty. Do it lockless to prevent lock bouncing. Races 2559 * are acceptable as soft limit is best effort anyway. 2560 */ 2561 if (RB_EMPTY_ROOT(&mctz->rb_root)) 2562 return 0; 2563 2564 /* 2565 * This loop can run a while, specially if mem_cgroup's continuously 2566 * keep exceeding their soft limit and putting the system under 2567 * pressure 2568 */ 2569 do { 2570 if (next_mz) 2571 mz = next_mz; 2572 else 2573 mz = mem_cgroup_largest_soft_limit_node(mctz); 2574 if (!mz) 2575 break; 2576 2577 nr_scanned = 0; 2578 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat, 2579 gfp_mask, &nr_scanned); 2580 nr_reclaimed += reclaimed; 2581 *total_scanned += nr_scanned; 2582 spin_lock_irq(&mctz->lock); 2583 __mem_cgroup_remove_exceeded(mz, mctz); 2584 2585 /* 2586 * If we failed to reclaim anything from this memory cgroup 2587 * it is time to move on to the next cgroup 2588 */ 2589 next_mz = NULL; 2590 if (!reclaimed) 2591 next_mz = __mem_cgroup_largest_soft_limit_node(mctz); 2592 2593 excess = soft_limit_excess(mz->memcg); 2594 /* 2595 * One school of thought says that we should not add 2596 * back the node to the tree if reclaim returns 0. 2597 * But our reclaim could return 0, simply because due 2598 * to priority we are exposing a smaller subset of 2599 * memory to reclaim from. Consider this as a longer 2600 * term TODO. 2601 */ 2602 /* If excess == 0, no tree ops */ 2603 __mem_cgroup_insert_exceeded(mz, mctz, excess); 2604 spin_unlock_irq(&mctz->lock); 2605 css_put(&mz->memcg->css); 2606 loop++; 2607 /* 2608 * Could not reclaim anything and there are no more 2609 * mem cgroups to try or we seem to be looping without 2610 * reclaiming anything. 2611 */ 2612 if (!nr_reclaimed && 2613 (next_mz == NULL || 2614 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS)) 2615 break; 2616 } while (!nr_reclaimed); 2617 if (next_mz) 2618 css_put(&next_mz->memcg->css); 2619 return nr_reclaimed; 2620 } 2621 2622 /* 2623 * Test whether @memcg has children, dead or alive. Note that this 2624 * function doesn't care whether @memcg has use_hierarchy enabled and 2625 * returns %true if there are child csses according to the cgroup 2626 * hierarchy. Testing use_hierarchy is the caller's responsiblity. 2627 */ 2628 static inline bool memcg_has_children(struct mem_cgroup *memcg) 2629 { 2630 bool ret; 2631 2632 rcu_read_lock(); 2633 ret = css_next_child(NULL, &memcg->css); 2634 rcu_read_unlock(); 2635 return ret; 2636 } 2637 2638 /* 2639 * Reclaims as many pages from the given memcg as possible. 2640 * 2641 * Caller is responsible for holding css reference for memcg. 2642 */ 2643 static int mem_cgroup_force_empty(struct mem_cgroup *memcg) 2644 { 2645 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES; 2646 2647 /* we call try-to-free pages for make this cgroup empty */ 2648 lru_add_drain_all(); 2649 /* try to free all pages in this cgroup */ 2650 while (nr_retries && page_counter_read(&memcg->memory)) { 2651 int progress; 2652 2653 if (signal_pending(current)) 2654 return -EINTR; 2655 2656 progress = try_to_free_mem_cgroup_pages(memcg, 1, 2657 GFP_KERNEL, true); 2658 if (!progress) { 2659 nr_retries--; 2660 /* maybe some writeback is necessary */ 2661 congestion_wait(BLK_RW_ASYNC, HZ/10); 2662 } 2663 2664 } 2665 2666 return 0; 2667 } 2668 2669 static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of, 2670 char *buf, size_t nbytes, 2671 loff_t off) 2672 { 2673 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 2674 2675 if (mem_cgroup_is_root(memcg)) 2676 return -EINVAL; 2677 return mem_cgroup_force_empty(memcg) ?: nbytes; 2678 } 2679 2680 static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css, 2681 struct cftype *cft) 2682 { 2683 return mem_cgroup_from_css(css)->use_hierarchy; 2684 } 2685 2686 static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css, 2687 struct cftype *cft, u64 val) 2688 { 2689 int retval = 0; 2690 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 2691 struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent); 2692 2693 if (memcg->use_hierarchy == val) 2694 return 0; 2695 2696 /* 2697 * If parent's use_hierarchy is set, we can't make any modifications 2698 * in the child subtrees. If it is unset, then the change can 2699 * occur, provided the current cgroup has no children. 2700 * 2701 * For the root cgroup, parent_mem is NULL, we allow value to be 2702 * set if there are no children. 2703 */ 2704 if ((!parent_memcg || !parent_memcg->use_hierarchy) && 2705 (val == 1 || val == 0)) { 2706 if (!memcg_has_children(memcg)) 2707 memcg->use_hierarchy = val; 2708 else 2709 retval = -EBUSY; 2710 } else 2711 retval = -EINVAL; 2712 2713 return retval; 2714 } 2715 2716 static void tree_stat(struct mem_cgroup *memcg, unsigned long *stat) 2717 { 2718 struct mem_cgroup *iter; 2719 int i; 2720 2721 memset(stat, 0, sizeof(*stat) * MEMCG_NR_STAT); 2722 2723 for_each_mem_cgroup_tree(iter, memcg) { 2724 for (i = 0; i < MEMCG_NR_STAT; i++) 2725 stat[i] += mem_cgroup_read_stat(iter, i); 2726 } 2727 } 2728 2729 static void tree_events(struct mem_cgroup *memcg, unsigned long *events) 2730 { 2731 struct mem_cgroup *iter; 2732 int i; 2733 2734 memset(events, 0, sizeof(*events) * MEMCG_NR_EVENTS); 2735 2736 for_each_mem_cgroup_tree(iter, memcg) { 2737 for (i = 0; i < MEMCG_NR_EVENTS; i++) 2738 events[i] += mem_cgroup_read_events(iter, i); 2739 } 2740 } 2741 2742 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap) 2743 { 2744 unsigned long val = 0; 2745 2746 if (mem_cgroup_is_root(memcg)) { 2747 struct mem_cgroup *iter; 2748 2749 for_each_mem_cgroup_tree(iter, memcg) { 2750 val += mem_cgroup_read_stat(iter, 2751 MEM_CGROUP_STAT_CACHE); 2752 val += mem_cgroup_read_stat(iter, 2753 MEM_CGROUP_STAT_RSS); 2754 if (swap) 2755 val += mem_cgroup_read_stat(iter, 2756 MEM_CGROUP_STAT_SWAP); 2757 } 2758 } else { 2759 if (!swap) 2760 val = page_counter_read(&memcg->memory); 2761 else 2762 val = page_counter_read(&memcg->memsw); 2763 } 2764 return val; 2765 } 2766 2767 enum { 2768 RES_USAGE, 2769 RES_LIMIT, 2770 RES_MAX_USAGE, 2771 RES_FAILCNT, 2772 RES_SOFT_LIMIT, 2773 }; 2774 2775 static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css, 2776 struct cftype *cft) 2777 { 2778 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 2779 struct page_counter *counter; 2780 2781 switch (MEMFILE_TYPE(cft->private)) { 2782 case _MEM: 2783 counter = &memcg->memory; 2784 break; 2785 case _MEMSWAP: 2786 counter = &memcg->memsw; 2787 break; 2788 case _KMEM: 2789 counter = &memcg->kmem; 2790 break; 2791 case _TCP: 2792 counter = &memcg->tcpmem; 2793 break; 2794 default: 2795 BUG(); 2796 } 2797 2798 switch (MEMFILE_ATTR(cft->private)) { 2799 case RES_USAGE: 2800 if (counter == &memcg->memory) 2801 return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE; 2802 if (counter == &memcg->memsw) 2803 return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE; 2804 return (u64)page_counter_read(counter) * PAGE_SIZE; 2805 case RES_LIMIT: 2806 return (u64)counter->limit * PAGE_SIZE; 2807 case RES_MAX_USAGE: 2808 return (u64)counter->watermark * PAGE_SIZE; 2809 case RES_FAILCNT: 2810 return counter->failcnt; 2811 case RES_SOFT_LIMIT: 2812 return (u64)memcg->soft_limit * PAGE_SIZE; 2813 default: 2814 BUG(); 2815 } 2816 } 2817 2818 #ifndef CONFIG_SLOB 2819 static int memcg_online_kmem(struct mem_cgroup *memcg) 2820 { 2821 int memcg_id; 2822 2823 if (cgroup_memory_nokmem) 2824 return 0; 2825 2826 BUG_ON(memcg->kmemcg_id >= 0); 2827 BUG_ON(memcg->kmem_state); 2828 2829 memcg_id = memcg_alloc_cache_id(); 2830 if (memcg_id < 0) 2831 return memcg_id; 2832 2833 static_branch_inc(&memcg_kmem_enabled_key); 2834 /* 2835 * A memory cgroup is considered kmem-online as soon as it gets 2836 * kmemcg_id. Setting the id after enabling static branching will 2837 * guarantee no one starts accounting before all call sites are 2838 * patched. 2839 */ 2840 memcg->kmemcg_id = memcg_id; 2841 memcg->kmem_state = KMEM_ONLINE; 2842 2843 return 0; 2844 } 2845 2846 static void memcg_offline_kmem(struct mem_cgroup *memcg) 2847 { 2848 struct cgroup_subsys_state *css; 2849 struct mem_cgroup *parent, *child; 2850 int kmemcg_id; 2851 2852 if (memcg->kmem_state != KMEM_ONLINE) 2853 return; 2854 /* 2855 * Clear the online state before clearing memcg_caches array 2856 * entries. The slab_mutex in memcg_deactivate_kmem_caches() 2857 * guarantees that no cache will be created for this cgroup 2858 * after we are done (see memcg_create_kmem_cache()). 2859 */ 2860 memcg->kmem_state = KMEM_ALLOCATED; 2861 2862 memcg_deactivate_kmem_caches(memcg); 2863 2864 kmemcg_id = memcg->kmemcg_id; 2865 BUG_ON(kmemcg_id < 0); 2866 2867 parent = parent_mem_cgroup(memcg); 2868 if (!parent) 2869 parent = root_mem_cgroup; 2870 2871 /* 2872 * Change kmemcg_id of this cgroup and all its descendants to the 2873 * parent's id, and then move all entries from this cgroup's list_lrus 2874 * to ones of the parent. After we have finished, all list_lrus 2875 * corresponding to this cgroup are guaranteed to remain empty. The 2876 * ordering is imposed by list_lru_node->lock taken by 2877 * memcg_drain_all_list_lrus(). 2878 */ 2879 rcu_read_lock(); /* can be called from css_free w/o cgroup_mutex */ 2880 css_for_each_descendant_pre(css, &memcg->css) { 2881 child = mem_cgroup_from_css(css); 2882 BUG_ON(child->kmemcg_id != kmemcg_id); 2883 child->kmemcg_id = parent->kmemcg_id; 2884 if (!memcg->use_hierarchy) 2885 break; 2886 } 2887 rcu_read_unlock(); 2888 2889 memcg_drain_all_list_lrus(kmemcg_id, parent->kmemcg_id); 2890 2891 memcg_free_cache_id(kmemcg_id); 2892 } 2893 2894 static void memcg_free_kmem(struct mem_cgroup *memcg) 2895 { 2896 /* css_alloc() failed, offlining didn't happen */ 2897 if (unlikely(memcg->kmem_state == KMEM_ONLINE)) 2898 memcg_offline_kmem(memcg); 2899 2900 if (memcg->kmem_state == KMEM_ALLOCATED) { 2901 memcg_destroy_kmem_caches(memcg); 2902 static_branch_dec(&memcg_kmem_enabled_key); 2903 WARN_ON(page_counter_read(&memcg->kmem)); 2904 } 2905 } 2906 #else 2907 static int memcg_online_kmem(struct mem_cgroup *memcg) 2908 { 2909 return 0; 2910 } 2911 static void memcg_offline_kmem(struct mem_cgroup *memcg) 2912 { 2913 } 2914 static void memcg_free_kmem(struct mem_cgroup *memcg) 2915 { 2916 } 2917 #endif /* !CONFIG_SLOB */ 2918 2919 static int memcg_update_kmem_limit(struct mem_cgroup *memcg, 2920 unsigned long limit) 2921 { 2922 int ret; 2923 2924 mutex_lock(&memcg_limit_mutex); 2925 ret = page_counter_limit(&memcg->kmem, limit); 2926 mutex_unlock(&memcg_limit_mutex); 2927 return ret; 2928 } 2929 2930 static int memcg_update_tcp_limit(struct mem_cgroup *memcg, unsigned long limit) 2931 { 2932 int ret; 2933 2934 mutex_lock(&memcg_limit_mutex); 2935 2936 ret = page_counter_limit(&memcg->tcpmem, limit); 2937 if (ret) 2938 goto out; 2939 2940 if (!memcg->tcpmem_active) { 2941 /* 2942 * The active flag needs to be written after the static_key 2943 * update. This is what guarantees that the socket activation 2944 * function is the last one to run. See mem_cgroup_sk_alloc() 2945 * for details, and note that we don't mark any socket as 2946 * belonging to this memcg until that flag is up. 2947 * 2948 * We need to do this, because static_keys will span multiple 2949 * sites, but we can't control their order. If we mark a socket 2950 * as accounted, but the accounting functions are not patched in 2951 * yet, we'll lose accounting. 2952 * 2953 * We never race with the readers in mem_cgroup_sk_alloc(), 2954 * because when this value change, the code to process it is not 2955 * patched in yet. 2956 */ 2957 static_branch_inc(&memcg_sockets_enabled_key); 2958 memcg->tcpmem_active = true; 2959 } 2960 out: 2961 mutex_unlock(&memcg_limit_mutex); 2962 return ret; 2963 } 2964 2965 /* 2966 * The user of this function is... 2967 * RES_LIMIT. 2968 */ 2969 static ssize_t mem_cgroup_write(struct kernfs_open_file *of, 2970 char *buf, size_t nbytes, loff_t off) 2971 { 2972 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 2973 unsigned long nr_pages; 2974 int ret; 2975 2976 buf = strstrip(buf); 2977 ret = page_counter_memparse(buf, "-1", &nr_pages); 2978 if (ret) 2979 return ret; 2980 2981 switch (MEMFILE_ATTR(of_cft(of)->private)) { 2982 case RES_LIMIT: 2983 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */ 2984 ret = -EINVAL; 2985 break; 2986 } 2987 switch (MEMFILE_TYPE(of_cft(of)->private)) { 2988 case _MEM: 2989 ret = mem_cgroup_resize_limit(memcg, nr_pages); 2990 break; 2991 case _MEMSWAP: 2992 ret = mem_cgroup_resize_memsw_limit(memcg, nr_pages); 2993 break; 2994 case _KMEM: 2995 ret = memcg_update_kmem_limit(memcg, nr_pages); 2996 break; 2997 case _TCP: 2998 ret = memcg_update_tcp_limit(memcg, nr_pages); 2999 break; 3000 } 3001 break; 3002 case RES_SOFT_LIMIT: 3003 memcg->soft_limit = nr_pages; 3004 ret = 0; 3005 break; 3006 } 3007 return ret ?: nbytes; 3008 } 3009 3010 static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf, 3011 size_t nbytes, loff_t off) 3012 { 3013 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 3014 struct page_counter *counter; 3015 3016 switch (MEMFILE_TYPE(of_cft(of)->private)) { 3017 case _MEM: 3018 counter = &memcg->memory; 3019 break; 3020 case _MEMSWAP: 3021 counter = &memcg->memsw; 3022 break; 3023 case _KMEM: 3024 counter = &memcg->kmem; 3025 break; 3026 case _TCP: 3027 counter = &memcg->tcpmem; 3028 break; 3029 default: 3030 BUG(); 3031 } 3032 3033 switch (MEMFILE_ATTR(of_cft(of)->private)) { 3034 case RES_MAX_USAGE: 3035 page_counter_reset_watermark(counter); 3036 break; 3037 case RES_FAILCNT: 3038 counter->failcnt = 0; 3039 break; 3040 default: 3041 BUG(); 3042 } 3043 3044 return nbytes; 3045 } 3046 3047 static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css, 3048 struct cftype *cft) 3049 { 3050 return mem_cgroup_from_css(css)->move_charge_at_immigrate; 3051 } 3052 3053 #ifdef CONFIG_MMU 3054 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css, 3055 struct cftype *cft, u64 val) 3056 { 3057 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 3058 3059 if (val & ~MOVE_MASK) 3060 return -EINVAL; 3061 3062 /* 3063 * No kind of locking is needed in here, because ->can_attach() will 3064 * check this value once in the beginning of the process, and then carry 3065 * on with stale data. This means that changes to this value will only 3066 * affect task migrations starting after the change. 3067 */ 3068 memcg->move_charge_at_immigrate = val; 3069 return 0; 3070 } 3071 #else 3072 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css, 3073 struct cftype *cft, u64 val) 3074 { 3075 return -ENOSYS; 3076 } 3077 #endif 3078 3079 #ifdef CONFIG_NUMA 3080 static int memcg_numa_stat_show(struct seq_file *m, void *v) 3081 { 3082 struct numa_stat { 3083 const char *name; 3084 unsigned int lru_mask; 3085 }; 3086 3087 static const struct numa_stat stats[] = { 3088 { "total", LRU_ALL }, 3089 { "file", LRU_ALL_FILE }, 3090 { "anon", LRU_ALL_ANON }, 3091 { "unevictable", BIT(LRU_UNEVICTABLE) }, 3092 }; 3093 const struct numa_stat *stat; 3094 int nid; 3095 unsigned long nr; 3096 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 3097 3098 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) { 3099 nr = mem_cgroup_nr_lru_pages(memcg, stat->lru_mask); 3100 seq_printf(m, "%s=%lu", stat->name, nr); 3101 for_each_node_state(nid, N_MEMORY) { 3102 nr = mem_cgroup_node_nr_lru_pages(memcg, nid, 3103 stat->lru_mask); 3104 seq_printf(m, " N%d=%lu", nid, nr); 3105 } 3106 seq_putc(m, '\n'); 3107 } 3108 3109 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) { 3110 struct mem_cgroup *iter; 3111 3112 nr = 0; 3113 for_each_mem_cgroup_tree(iter, memcg) 3114 nr += mem_cgroup_nr_lru_pages(iter, stat->lru_mask); 3115 seq_printf(m, "hierarchical_%s=%lu", stat->name, nr); 3116 for_each_node_state(nid, N_MEMORY) { 3117 nr = 0; 3118 for_each_mem_cgroup_tree(iter, memcg) 3119 nr += mem_cgroup_node_nr_lru_pages( 3120 iter, nid, stat->lru_mask); 3121 seq_printf(m, " N%d=%lu", nid, nr); 3122 } 3123 seq_putc(m, '\n'); 3124 } 3125 3126 return 0; 3127 } 3128 #endif /* CONFIG_NUMA */ 3129 3130 static int memcg_stat_show(struct seq_file *m, void *v) 3131 { 3132 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 3133 unsigned long memory, memsw; 3134 struct mem_cgroup *mi; 3135 unsigned int i; 3136 3137 BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_stat_names) != 3138 MEM_CGROUP_STAT_NSTATS); 3139 BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_events_names) != 3140 MEM_CGROUP_EVENTS_NSTATS); 3141 BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS); 3142 3143 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) { 3144 if (i == MEM_CGROUP_STAT_SWAP && !do_memsw_account()) 3145 continue; 3146 seq_printf(m, "%s %lu\n", mem_cgroup_stat_names[i], 3147 mem_cgroup_read_stat(memcg, i) * PAGE_SIZE); 3148 } 3149 3150 for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) 3151 seq_printf(m, "%s %lu\n", mem_cgroup_events_names[i], 3152 mem_cgroup_read_events(memcg, i)); 3153 3154 for (i = 0; i < NR_LRU_LISTS; i++) 3155 seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i], 3156 mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE); 3157 3158 /* Hierarchical information */ 3159 memory = memsw = PAGE_COUNTER_MAX; 3160 for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) { 3161 memory = min(memory, mi->memory.limit); 3162 memsw = min(memsw, mi->memsw.limit); 3163 } 3164 seq_printf(m, "hierarchical_memory_limit %llu\n", 3165 (u64)memory * PAGE_SIZE); 3166 if (do_memsw_account()) 3167 seq_printf(m, "hierarchical_memsw_limit %llu\n", 3168 (u64)memsw * PAGE_SIZE); 3169 3170 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) { 3171 unsigned long long val = 0; 3172 3173 if (i == MEM_CGROUP_STAT_SWAP && !do_memsw_account()) 3174 continue; 3175 for_each_mem_cgroup_tree(mi, memcg) 3176 val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE; 3177 seq_printf(m, "total_%s %llu\n", mem_cgroup_stat_names[i], val); 3178 } 3179 3180 for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) { 3181 unsigned long long val = 0; 3182 3183 for_each_mem_cgroup_tree(mi, memcg) 3184 val += mem_cgroup_read_events(mi, i); 3185 seq_printf(m, "total_%s %llu\n", 3186 mem_cgroup_events_names[i], val); 3187 } 3188 3189 for (i = 0; i < NR_LRU_LISTS; i++) { 3190 unsigned long long val = 0; 3191 3192 for_each_mem_cgroup_tree(mi, memcg) 3193 val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE; 3194 seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val); 3195 } 3196 3197 #ifdef CONFIG_DEBUG_VM 3198 { 3199 pg_data_t *pgdat; 3200 struct mem_cgroup_per_node *mz; 3201 struct zone_reclaim_stat *rstat; 3202 unsigned long recent_rotated[2] = {0, 0}; 3203 unsigned long recent_scanned[2] = {0, 0}; 3204 3205 for_each_online_pgdat(pgdat) { 3206 mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id); 3207 rstat = &mz->lruvec.reclaim_stat; 3208 3209 recent_rotated[0] += rstat->recent_rotated[0]; 3210 recent_rotated[1] += rstat->recent_rotated[1]; 3211 recent_scanned[0] += rstat->recent_scanned[0]; 3212 recent_scanned[1] += rstat->recent_scanned[1]; 3213 } 3214 seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]); 3215 seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]); 3216 seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]); 3217 seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]); 3218 } 3219 #endif 3220 3221 return 0; 3222 } 3223 3224 static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css, 3225 struct cftype *cft) 3226 { 3227 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 3228 3229 return mem_cgroup_swappiness(memcg); 3230 } 3231 3232 static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css, 3233 struct cftype *cft, u64 val) 3234 { 3235 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 3236 3237 if (val > 100) 3238 return -EINVAL; 3239 3240 if (css->parent) 3241 memcg->swappiness = val; 3242 else 3243 vm_swappiness = val; 3244 3245 return 0; 3246 } 3247 3248 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap) 3249 { 3250 struct mem_cgroup_threshold_ary *t; 3251 unsigned long usage; 3252 int i; 3253 3254 rcu_read_lock(); 3255 if (!swap) 3256 t = rcu_dereference(memcg->thresholds.primary); 3257 else 3258 t = rcu_dereference(memcg->memsw_thresholds.primary); 3259 3260 if (!t) 3261 goto unlock; 3262 3263 usage = mem_cgroup_usage(memcg, swap); 3264 3265 /* 3266 * current_threshold points to threshold just below or equal to usage. 3267 * If it's not true, a threshold was crossed after last 3268 * call of __mem_cgroup_threshold(). 3269 */ 3270 i = t->current_threshold; 3271 3272 /* 3273 * Iterate backward over array of thresholds starting from 3274 * current_threshold and check if a threshold is crossed. 3275 * If none of thresholds below usage is crossed, we read 3276 * only one element of the array here. 3277 */ 3278 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--) 3279 eventfd_signal(t->entries[i].eventfd, 1); 3280 3281 /* i = current_threshold + 1 */ 3282 i++; 3283 3284 /* 3285 * Iterate forward over array of thresholds starting from 3286 * current_threshold+1 and check if a threshold is crossed. 3287 * If none of thresholds above usage is crossed, we read 3288 * only one element of the array here. 3289 */ 3290 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++) 3291 eventfd_signal(t->entries[i].eventfd, 1); 3292 3293 /* Update current_threshold */ 3294 t->current_threshold = i - 1; 3295 unlock: 3296 rcu_read_unlock(); 3297 } 3298 3299 static void mem_cgroup_threshold(struct mem_cgroup *memcg) 3300 { 3301 while (memcg) { 3302 __mem_cgroup_threshold(memcg, false); 3303 if (do_memsw_account()) 3304 __mem_cgroup_threshold(memcg, true); 3305 3306 memcg = parent_mem_cgroup(memcg); 3307 } 3308 } 3309 3310 static int compare_thresholds(const void *a, const void *b) 3311 { 3312 const struct mem_cgroup_threshold *_a = a; 3313 const struct mem_cgroup_threshold *_b = b; 3314 3315 if (_a->threshold > _b->threshold) 3316 return 1; 3317 3318 if (_a->threshold < _b->threshold) 3319 return -1; 3320 3321 return 0; 3322 } 3323 3324 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg) 3325 { 3326 struct mem_cgroup_eventfd_list *ev; 3327 3328 spin_lock(&memcg_oom_lock); 3329 3330 list_for_each_entry(ev, &memcg->oom_notify, list) 3331 eventfd_signal(ev->eventfd, 1); 3332 3333 spin_unlock(&memcg_oom_lock); 3334 return 0; 3335 } 3336 3337 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg) 3338 { 3339 struct mem_cgroup *iter; 3340 3341 for_each_mem_cgroup_tree(iter, memcg) 3342 mem_cgroup_oom_notify_cb(iter); 3343 } 3344 3345 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg, 3346 struct eventfd_ctx *eventfd, const char *args, enum res_type type) 3347 { 3348 struct mem_cgroup_thresholds *thresholds; 3349 struct mem_cgroup_threshold_ary *new; 3350 unsigned long threshold; 3351 unsigned long usage; 3352 int i, size, ret; 3353 3354 ret = page_counter_memparse(args, "-1", &threshold); 3355 if (ret) 3356 return ret; 3357 3358 mutex_lock(&memcg->thresholds_lock); 3359 3360 if (type == _MEM) { 3361 thresholds = &memcg->thresholds; 3362 usage = mem_cgroup_usage(memcg, false); 3363 } else if (type == _MEMSWAP) { 3364 thresholds = &memcg->memsw_thresholds; 3365 usage = mem_cgroup_usage(memcg, true); 3366 } else 3367 BUG(); 3368 3369 /* Check if a threshold crossed before adding a new one */ 3370 if (thresholds->primary) 3371 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 3372 3373 size = thresholds->primary ? thresholds->primary->size + 1 : 1; 3374 3375 /* Allocate memory for new array of thresholds */ 3376 new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold), 3377 GFP_KERNEL); 3378 if (!new) { 3379 ret = -ENOMEM; 3380 goto unlock; 3381 } 3382 new->size = size; 3383 3384 /* Copy thresholds (if any) to new array */ 3385 if (thresholds->primary) { 3386 memcpy(new->entries, thresholds->primary->entries, (size - 1) * 3387 sizeof(struct mem_cgroup_threshold)); 3388 } 3389 3390 /* Add new threshold */ 3391 new->entries[size - 1].eventfd = eventfd; 3392 new->entries[size - 1].threshold = threshold; 3393 3394 /* Sort thresholds. Registering of new threshold isn't time-critical */ 3395 sort(new->entries, size, sizeof(struct mem_cgroup_threshold), 3396 compare_thresholds, NULL); 3397 3398 /* Find current threshold */ 3399 new->current_threshold = -1; 3400 for (i = 0; i < size; i++) { 3401 if (new->entries[i].threshold <= usage) { 3402 /* 3403 * new->current_threshold will not be used until 3404 * rcu_assign_pointer(), so it's safe to increment 3405 * it here. 3406 */ 3407 ++new->current_threshold; 3408 } else 3409 break; 3410 } 3411 3412 /* Free old spare buffer and save old primary buffer as spare */ 3413 kfree(thresholds->spare); 3414 thresholds->spare = thresholds->primary; 3415 3416 rcu_assign_pointer(thresholds->primary, new); 3417 3418 /* To be sure that nobody uses thresholds */ 3419 synchronize_rcu(); 3420 3421 unlock: 3422 mutex_unlock(&memcg->thresholds_lock); 3423 3424 return ret; 3425 } 3426 3427 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg, 3428 struct eventfd_ctx *eventfd, const char *args) 3429 { 3430 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM); 3431 } 3432 3433 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg, 3434 struct eventfd_ctx *eventfd, const char *args) 3435 { 3436 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP); 3437 } 3438 3439 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, 3440 struct eventfd_ctx *eventfd, enum res_type type) 3441 { 3442 struct mem_cgroup_thresholds *thresholds; 3443 struct mem_cgroup_threshold_ary *new; 3444 unsigned long usage; 3445 int i, j, size; 3446 3447 mutex_lock(&memcg->thresholds_lock); 3448 3449 if (type == _MEM) { 3450 thresholds = &memcg->thresholds; 3451 usage = mem_cgroup_usage(memcg, false); 3452 } else if (type == _MEMSWAP) { 3453 thresholds = &memcg->memsw_thresholds; 3454 usage = mem_cgroup_usage(memcg, true); 3455 } else 3456 BUG(); 3457 3458 if (!thresholds->primary) 3459 goto unlock; 3460 3461 /* Check if a threshold crossed before removing */ 3462 __mem_cgroup_threshold(memcg, type == _MEMSWAP); 3463 3464 /* Calculate new number of threshold */ 3465 size = 0; 3466 for (i = 0; i < thresholds->primary->size; i++) { 3467 if (thresholds->primary->entries[i].eventfd != eventfd) 3468 size++; 3469 } 3470 3471 new = thresholds->spare; 3472 3473 /* Set thresholds array to NULL if we don't have thresholds */ 3474 if (!size) { 3475 kfree(new); 3476 new = NULL; 3477 goto swap_buffers; 3478 } 3479 3480 new->size = size; 3481 3482 /* Copy thresholds and find current threshold */ 3483 new->current_threshold = -1; 3484 for (i = 0, j = 0; i < thresholds->primary->size; i++) { 3485 if (thresholds->primary->entries[i].eventfd == eventfd) 3486 continue; 3487 3488 new->entries[j] = thresholds->primary->entries[i]; 3489 if (new->entries[j].threshold <= usage) { 3490 /* 3491 * new->current_threshold will not be used 3492 * until rcu_assign_pointer(), so it's safe to increment 3493 * it here. 3494 */ 3495 ++new->current_threshold; 3496 } 3497 j++; 3498 } 3499 3500 swap_buffers: 3501 /* Swap primary and spare array */ 3502 thresholds->spare = thresholds->primary; 3503 3504 rcu_assign_pointer(thresholds->primary, new); 3505 3506 /* To be sure that nobody uses thresholds */ 3507 synchronize_rcu(); 3508 3509 /* If all events are unregistered, free the spare array */ 3510 if (!new) { 3511 kfree(thresholds->spare); 3512 thresholds->spare = NULL; 3513 } 3514 unlock: 3515 mutex_unlock(&memcg->thresholds_lock); 3516 } 3517 3518 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, 3519 struct eventfd_ctx *eventfd) 3520 { 3521 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM); 3522 } 3523 3524 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg, 3525 struct eventfd_ctx *eventfd) 3526 { 3527 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP); 3528 } 3529 3530 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg, 3531 struct eventfd_ctx *eventfd, const char *args) 3532 { 3533 struct mem_cgroup_eventfd_list *event; 3534 3535 event = kmalloc(sizeof(*event), GFP_KERNEL); 3536 if (!event) 3537 return -ENOMEM; 3538 3539 spin_lock(&memcg_oom_lock); 3540 3541 event->eventfd = eventfd; 3542 list_add(&event->list, &memcg->oom_notify); 3543 3544 /* already in OOM ? */ 3545 if (memcg->under_oom) 3546 eventfd_signal(eventfd, 1); 3547 spin_unlock(&memcg_oom_lock); 3548 3549 return 0; 3550 } 3551 3552 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg, 3553 struct eventfd_ctx *eventfd) 3554 { 3555 struct mem_cgroup_eventfd_list *ev, *tmp; 3556 3557 spin_lock(&memcg_oom_lock); 3558 3559 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) { 3560 if (ev->eventfd == eventfd) { 3561 list_del(&ev->list); 3562 kfree(ev); 3563 } 3564 } 3565 3566 spin_unlock(&memcg_oom_lock); 3567 } 3568 3569 static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v) 3570 { 3571 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf)); 3572 3573 seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable); 3574 seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom); 3575 return 0; 3576 } 3577 3578 static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css, 3579 struct cftype *cft, u64 val) 3580 { 3581 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 3582 3583 /* cannot set to root cgroup and only 0 and 1 are allowed */ 3584 if (!css->parent || !((val == 0) || (val == 1))) 3585 return -EINVAL; 3586 3587 memcg->oom_kill_disable = val; 3588 if (!val) 3589 memcg_oom_recover(memcg); 3590 3591 return 0; 3592 } 3593 3594 #ifdef CONFIG_CGROUP_WRITEBACK 3595 3596 struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg) 3597 { 3598 return &memcg->cgwb_list; 3599 } 3600 3601 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp) 3602 { 3603 return wb_domain_init(&memcg->cgwb_domain, gfp); 3604 } 3605 3606 static void memcg_wb_domain_exit(struct mem_cgroup *memcg) 3607 { 3608 wb_domain_exit(&memcg->cgwb_domain); 3609 } 3610 3611 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg) 3612 { 3613 wb_domain_size_changed(&memcg->cgwb_domain); 3614 } 3615 3616 struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb) 3617 { 3618 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); 3619 3620 if (!memcg->css.parent) 3621 return NULL; 3622 3623 return &memcg->cgwb_domain; 3624 } 3625 3626 /** 3627 * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg 3628 * @wb: bdi_writeback in question 3629 * @pfilepages: out parameter for number of file pages 3630 * @pheadroom: out parameter for number of allocatable pages according to memcg 3631 * @pdirty: out parameter for number of dirty pages 3632 * @pwriteback: out parameter for number of pages under writeback 3633 * 3634 * Determine the numbers of file, headroom, dirty, and writeback pages in 3635 * @wb's memcg. File, dirty and writeback are self-explanatory. Headroom 3636 * is a bit more involved. 3637 * 3638 * A memcg's headroom is "min(max, high) - used". In the hierarchy, the 3639 * headroom is calculated as the lowest headroom of itself and the 3640 * ancestors. Note that this doesn't consider the actual amount of 3641 * available memory in the system. The caller should further cap 3642 * *@pheadroom accordingly. 3643 */ 3644 void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages, 3645 unsigned long *pheadroom, unsigned long *pdirty, 3646 unsigned long *pwriteback) 3647 { 3648 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); 3649 struct mem_cgroup *parent; 3650 3651 *pdirty = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_DIRTY); 3652 3653 /* this should eventually include NR_UNSTABLE_NFS */ 3654 *pwriteback = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_WRITEBACK); 3655 *pfilepages = mem_cgroup_nr_lru_pages(memcg, (1 << LRU_INACTIVE_FILE) | 3656 (1 << LRU_ACTIVE_FILE)); 3657 *pheadroom = PAGE_COUNTER_MAX; 3658 3659 while ((parent = parent_mem_cgroup(memcg))) { 3660 unsigned long ceiling = min(memcg->memory.limit, memcg->high); 3661 unsigned long used = page_counter_read(&memcg->memory); 3662 3663 *pheadroom = min(*pheadroom, ceiling - min(ceiling, used)); 3664 memcg = parent; 3665 } 3666 } 3667 3668 #else /* CONFIG_CGROUP_WRITEBACK */ 3669 3670 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp) 3671 { 3672 return 0; 3673 } 3674 3675 static void memcg_wb_domain_exit(struct mem_cgroup *memcg) 3676 { 3677 } 3678 3679 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg) 3680 { 3681 } 3682 3683 #endif /* CONFIG_CGROUP_WRITEBACK */ 3684 3685 /* 3686 * DO NOT USE IN NEW FILES. 3687 * 3688 * "cgroup.event_control" implementation. 3689 * 3690 * This is way over-engineered. It tries to support fully configurable 3691 * events for each user. Such level of flexibility is completely 3692 * unnecessary especially in the light of the planned unified hierarchy. 3693 * 3694 * Please deprecate this and replace with something simpler if at all 3695 * possible. 3696 */ 3697 3698 /* 3699 * Unregister event and free resources. 3700 * 3701 * Gets called from workqueue. 3702 */ 3703 static void memcg_event_remove(struct work_struct *work) 3704 { 3705 struct mem_cgroup_event *event = 3706 container_of(work, struct mem_cgroup_event, remove); 3707 struct mem_cgroup *memcg = event->memcg; 3708 3709 remove_wait_queue(event->wqh, &event->wait); 3710 3711 event->unregister_event(memcg, event->eventfd); 3712 3713 /* Notify userspace the event is going away. */ 3714 eventfd_signal(event->eventfd, 1); 3715 3716 eventfd_ctx_put(event->eventfd); 3717 kfree(event); 3718 css_put(&memcg->css); 3719 } 3720 3721 /* 3722 * Gets called on POLLHUP on eventfd when user closes it. 3723 * 3724 * Called with wqh->lock held and interrupts disabled. 3725 */ 3726 static int memcg_event_wake(wait_queue_t *wait, unsigned mode, 3727 int sync, void *key) 3728 { 3729 struct mem_cgroup_event *event = 3730 container_of(wait, struct mem_cgroup_event, wait); 3731 struct mem_cgroup *memcg = event->memcg; 3732 unsigned long flags = (unsigned long)key; 3733 3734 if (flags & POLLHUP) { 3735 /* 3736 * If the event has been detached at cgroup removal, we 3737 * can simply return knowing the other side will cleanup 3738 * for us. 3739 * 3740 * We can't race against event freeing since the other 3741 * side will require wqh->lock via remove_wait_queue(), 3742 * which we hold. 3743 */ 3744 spin_lock(&memcg->event_list_lock); 3745 if (!list_empty(&event->list)) { 3746 list_del_init(&event->list); 3747 /* 3748 * We are in atomic context, but cgroup_event_remove() 3749 * may sleep, so we have to call it in workqueue. 3750 */ 3751 schedule_work(&event->remove); 3752 } 3753 spin_unlock(&memcg->event_list_lock); 3754 } 3755 3756 return 0; 3757 } 3758 3759 static void memcg_event_ptable_queue_proc(struct file *file, 3760 wait_queue_head_t *wqh, poll_table *pt) 3761 { 3762 struct mem_cgroup_event *event = 3763 container_of(pt, struct mem_cgroup_event, pt); 3764 3765 event->wqh = wqh; 3766 add_wait_queue(wqh, &event->wait); 3767 } 3768 3769 /* 3770 * DO NOT USE IN NEW FILES. 3771 * 3772 * Parse input and register new cgroup event handler. 3773 * 3774 * Input must be in format '<event_fd> <control_fd> <args>'. 3775 * Interpretation of args is defined by control file implementation. 3776 */ 3777 static ssize_t memcg_write_event_control(struct kernfs_open_file *of, 3778 char *buf, size_t nbytes, loff_t off) 3779 { 3780 struct cgroup_subsys_state *css = of_css(of); 3781 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 3782 struct mem_cgroup_event *event; 3783 struct cgroup_subsys_state *cfile_css; 3784 unsigned int efd, cfd; 3785 struct fd efile; 3786 struct fd cfile; 3787 const char *name; 3788 char *endp; 3789 int ret; 3790 3791 buf = strstrip(buf); 3792 3793 efd = simple_strtoul(buf, &endp, 10); 3794 if (*endp != ' ') 3795 return -EINVAL; 3796 buf = endp + 1; 3797 3798 cfd = simple_strtoul(buf, &endp, 10); 3799 if ((*endp != ' ') && (*endp != '\0')) 3800 return -EINVAL; 3801 buf = endp + 1; 3802 3803 event = kzalloc(sizeof(*event), GFP_KERNEL); 3804 if (!event) 3805 return -ENOMEM; 3806 3807 event->memcg = memcg; 3808 INIT_LIST_HEAD(&event->list); 3809 init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc); 3810 init_waitqueue_func_entry(&event->wait, memcg_event_wake); 3811 INIT_WORK(&event->remove, memcg_event_remove); 3812 3813 efile = fdget(efd); 3814 if (!efile.file) { 3815 ret = -EBADF; 3816 goto out_kfree; 3817 } 3818 3819 event->eventfd = eventfd_ctx_fileget(efile.file); 3820 if (IS_ERR(event->eventfd)) { 3821 ret = PTR_ERR(event->eventfd); 3822 goto out_put_efile; 3823 } 3824 3825 cfile = fdget(cfd); 3826 if (!cfile.file) { 3827 ret = -EBADF; 3828 goto out_put_eventfd; 3829 } 3830 3831 /* the process need read permission on control file */ 3832 /* AV: shouldn't we check that it's been opened for read instead? */ 3833 ret = inode_permission(file_inode(cfile.file), MAY_READ); 3834 if (ret < 0) 3835 goto out_put_cfile; 3836 3837 /* 3838 * Determine the event callbacks and set them in @event. This used 3839 * to be done via struct cftype but cgroup core no longer knows 3840 * about these events. The following is crude but the whole thing 3841 * is for compatibility anyway. 3842 * 3843 * DO NOT ADD NEW FILES. 3844 */ 3845 name = cfile.file->f_path.dentry->d_name.name; 3846 3847 if (!strcmp(name, "memory.usage_in_bytes")) { 3848 event->register_event = mem_cgroup_usage_register_event; 3849 event->unregister_event = mem_cgroup_usage_unregister_event; 3850 } else if (!strcmp(name, "memory.oom_control")) { 3851 event->register_event = mem_cgroup_oom_register_event; 3852 event->unregister_event = mem_cgroup_oom_unregister_event; 3853 } else if (!strcmp(name, "memory.pressure_level")) { 3854 event->register_event = vmpressure_register_event; 3855 event->unregister_event = vmpressure_unregister_event; 3856 } else if (!strcmp(name, "memory.memsw.usage_in_bytes")) { 3857 event->register_event = memsw_cgroup_usage_register_event; 3858 event->unregister_event = memsw_cgroup_usage_unregister_event; 3859 } else { 3860 ret = -EINVAL; 3861 goto out_put_cfile; 3862 } 3863 3864 /* 3865 * Verify @cfile should belong to @css. Also, remaining events are 3866 * automatically removed on cgroup destruction but the removal is 3867 * asynchronous, so take an extra ref on @css. 3868 */ 3869 cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent, 3870 &memory_cgrp_subsys); 3871 ret = -EINVAL; 3872 if (IS_ERR(cfile_css)) 3873 goto out_put_cfile; 3874 if (cfile_css != css) { 3875 css_put(cfile_css); 3876 goto out_put_cfile; 3877 } 3878 3879 ret = event->register_event(memcg, event->eventfd, buf); 3880 if (ret) 3881 goto out_put_css; 3882 3883 efile.file->f_op->poll(efile.file, &event->pt); 3884 3885 spin_lock(&memcg->event_list_lock); 3886 list_add(&event->list, &memcg->event_list); 3887 spin_unlock(&memcg->event_list_lock); 3888 3889 fdput(cfile); 3890 fdput(efile); 3891 3892 return nbytes; 3893 3894 out_put_css: 3895 css_put(css); 3896 out_put_cfile: 3897 fdput(cfile); 3898 out_put_eventfd: 3899 eventfd_ctx_put(event->eventfd); 3900 out_put_efile: 3901 fdput(efile); 3902 out_kfree: 3903 kfree(event); 3904 3905 return ret; 3906 } 3907 3908 static struct cftype mem_cgroup_legacy_files[] = { 3909 { 3910 .name = "usage_in_bytes", 3911 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE), 3912 .read_u64 = mem_cgroup_read_u64, 3913 }, 3914 { 3915 .name = "max_usage_in_bytes", 3916 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE), 3917 .write = mem_cgroup_reset, 3918 .read_u64 = mem_cgroup_read_u64, 3919 }, 3920 { 3921 .name = "limit_in_bytes", 3922 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT), 3923 .write = mem_cgroup_write, 3924 .read_u64 = mem_cgroup_read_u64, 3925 }, 3926 { 3927 .name = "soft_limit_in_bytes", 3928 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT), 3929 .write = mem_cgroup_write, 3930 .read_u64 = mem_cgroup_read_u64, 3931 }, 3932 { 3933 .name = "failcnt", 3934 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT), 3935 .write = mem_cgroup_reset, 3936 .read_u64 = mem_cgroup_read_u64, 3937 }, 3938 { 3939 .name = "stat", 3940 .seq_show = memcg_stat_show, 3941 }, 3942 { 3943 .name = "force_empty", 3944 .write = mem_cgroup_force_empty_write, 3945 }, 3946 { 3947 .name = "use_hierarchy", 3948 .write_u64 = mem_cgroup_hierarchy_write, 3949 .read_u64 = mem_cgroup_hierarchy_read, 3950 }, 3951 { 3952 .name = "cgroup.event_control", /* XXX: for compat */ 3953 .write = memcg_write_event_control, 3954 .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE, 3955 }, 3956 { 3957 .name = "swappiness", 3958 .read_u64 = mem_cgroup_swappiness_read, 3959 .write_u64 = mem_cgroup_swappiness_write, 3960 }, 3961 { 3962 .name = "move_charge_at_immigrate", 3963 .read_u64 = mem_cgroup_move_charge_read, 3964 .write_u64 = mem_cgroup_move_charge_write, 3965 }, 3966 { 3967 .name = "oom_control", 3968 .seq_show = mem_cgroup_oom_control_read, 3969 .write_u64 = mem_cgroup_oom_control_write, 3970 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL), 3971 }, 3972 { 3973 .name = "pressure_level", 3974 }, 3975 #ifdef CONFIG_NUMA 3976 { 3977 .name = "numa_stat", 3978 .seq_show = memcg_numa_stat_show, 3979 }, 3980 #endif 3981 { 3982 .name = "kmem.limit_in_bytes", 3983 .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT), 3984 .write = mem_cgroup_write, 3985 .read_u64 = mem_cgroup_read_u64, 3986 }, 3987 { 3988 .name = "kmem.usage_in_bytes", 3989 .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE), 3990 .read_u64 = mem_cgroup_read_u64, 3991 }, 3992 { 3993 .name = "kmem.failcnt", 3994 .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT), 3995 .write = mem_cgroup_reset, 3996 .read_u64 = mem_cgroup_read_u64, 3997 }, 3998 { 3999 .name = "kmem.max_usage_in_bytes", 4000 .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE), 4001 .write = mem_cgroup_reset, 4002 .read_u64 = mem_cgroup_read_u64, 4003 }, 4004 #ifdef CONFIG_SLABINFO 4005 { 4006 .name = "kmem.slabinfo", 4007 .seq_start = slab_start, 4008 .seq_next = slab_next, 4009 .seq_stop = slab_stop, 4010 .seq_show = memcg_slab_show, 4011 }, 4012 #endif 4013 { 4014 .name = "kmem.tcp.limit_in_bytes", 4015 .private = MEMFILE_PRIVATE(_TCP, RES_LIMIT), 4016 .write = mem_cgroup_write, 4017 .read_u64 = mem_cgroup_read_u64, 4018 }, 4019 { 4020 .name = "kmem.tcp.usage_in_bytes", 4021 .private = MEMFILE_PRIVATE(_TCP, RES_USAGE), 4022 .read_u64 = mem_cgroup_read_u64, 4023 }, 4024 { 4025 .name = "kmem.tcp.failcnt", 4026 .private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT), 4027 .write = mem_cgroup_reset, 4028 .read_u64 = mem_cgroup_read_u64, 4029 }, 4030 { 4031 .name = "kmem.tcp.max_usage_in_bytes", 4032 .private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE), 4033 .write = mem_cgroup_reset, 4034 .read_u64 = mem_cgroup_read_u64, 4035 }, 4036 { }, /* terminate */ 4037 }; 4038 4039 /* 4040 * Private memory cgroup IDR 4041 * 4042 * Swap-out records and page cache shadow entries need to store memcg 4043 * references in constrained space, so we maintain an ID space that is 4044 * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of 4045 * memory-controlled cgroups to 64k. 4046 * 4047 * However, there usually are many references to the oflline CSS after 4048 * the cgroup has been destroyed, such as page cache or reclaimable 4049 * slab objects, that don't need to hang on to the ID. We want to keep 4050 * those dead CSS from occupying IDs, or we might quickly exhaust the 4051 * relatively small ID space and prevent the creation of new cgroups 4052 * even when there are much fewer than 64k cgroups - possibly none. 4053 * 4054 * Maintain a private 16-bit ID space for memcg, and allow the ID to 4055 * be freed and recycled when it's no longer needed, which is usually 4056 * when the CSS is offlined. 4057 * 4058 * The only exception to that are records of swapped out tmpfs/shmem 4059 * pages that need to be attributed to live ancestors on swapin. But 4060 * those references are manageable from userspace. 4061 */ 4062 4063 static DEFINE_IDR(mem_cgroup_idr); 4064 4065 static void mem_cgroup_id_get_many(struct mem_cgroup *memcg, unsigned int n) 4066 { 4067 VM_BUG_ON(atomic_read(&memcg->id.ref) <= 0); 4068 atomic_add(n, &memcg->id.ref); 4069 } 4070 4071 static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n) 4072 { 4073 VM_BUG_ON(atomic_read(&memcg->id.ref) < n); 4074 if (atomic_sub_and_test(n, &memcg->id.ref)) { 4075 idr_remove(&mem_cgroup_idr, memcg->id.id); 4076 memcg->id.id = 0; 4077 4078 /* Memcg ID pins CSS */ 4079 css_put(&memcg->css); 4080 } 4081 } 4082 4083 static inline void mem_cgroup_id_get(struct mem_cgroup *memcg) 4084 { 4085 mem_cgroup_id_get_many(memcg, 1); 4086 } 4087 4088 static inline void mem_cgroup_id_put(struct mem_cgroup *memcg) 4089 { 4090 mem_cgroup_id_put_many(memcg, 1); 4091 } 4092 4093 /** 4094 * mem_cgroup_from_id - look up a memcg from a memcg id 4095 * @id: the memcg id to look up 4096 * 4097 * Caller must hold rcu_read_lock(). 4098 */ 4099 struct mem_cgroup *mem_cgroup_from_id(unsigned short id) 4100 { 4101 WARN_ON_ONCE(!rcu_read_lock_held()); 4102 return idr_find(&mem_cgroup_idr, id); 4103 } 4104 4105 static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) 4106 { 4107 struct mem_cgroup_per_node *pn; 4108 int tmp = node; 4109 /* 4110 * This routine is called against possible nodes. 4111 * But it's BUG to call kmalloc() against offline node. 4112 * 4113 * TODO: this routine can waste much memory for nodes which will 4114 * never be onlined. It's better to use memory hotplug callback 4115 * function. 4116 */ 4117 if (!node_state(node, N_NORMAL_MEMORY)) 4118 tmp = -1; 4119 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp); 4120 if (!pn) 4121 return 1; 4122 4123 lruvec_init(&pn->lruvec); 4124 pn->usage_in_excess = 0; 4125 pn->on_tree = false; 4126 pn->memcg = memcg; 4127 4128 memcg->nodeinfo[node] = pn; 4129 return 0; 4130 } 4131 4132 static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) 4133 { 4134 kfree(memcg->nodeinfo[node]); 4135 } 4136 4137 static void mem_cgroup_free(struct mem_cgroup *memcg) 4138 { 4139 int node; 4140 4141 memcg_wb_domain_exit(memcg); 4142 for_each_node(node) 4143 free_mem_cgroup_per_node_info(memcg, node); 4144 free_percpu(memcg->stat); 4145 kfree(memcg); 4146 } 4147 4148 static struct mem_cgroup *mem_cgroup_alloc(void) 4149 { 4150 struct mem_cgroup *memcg; 4151 size_t size; 4152 int node; 4153 4154 size = sizeof(struct mem_cgroup); 4155 size += nr_node_ids * sizeof(struct mem_cgroup_per_node *); 4156 4157 memcg = kzalloc(size, GFP_KERNEL); 4158 if (!memcg) 4159 return NULL; 4160 4161 memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL, 4162 1, MEM_CGROUP_ID_MAX, 4163 GFP_KERNEL); 4164 if (memcg->id.id < 0) 4165 goto fail; 4166 4167 memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu); 4168 if (!memcg->stat) 4169 goto fail; 4170 4171 for_each_node(node) 4172 if (alloc_mem_cgroup_per_node_info(memcg, node)) 4173 goto fail; 4174 4175 if (memcg_wb_domain_init(memcg, GFP_KERNEL)) 4176 goto fail; 4177 4178 INIT_WORK(&memcg->high_work, high_work_func); 4179 memcg->last_scanned_node = MAX_NUMNODES; 4180 INIT_LIST_HEAD(&memcg->oom_notify); 4181 mutex_init(&memcg->thresholds_lock); 4182 spin_lock_init(&memcg->move_lock); 4183 vmpressure_init(&memcg->vmpressure); 4184 INIT_LIST_HEAD(&memcg->event_list); 4185 spin_lock_init(&memcg->event_list_lock); 4186 memcg->socket_pressure = jiffies; 4187 #ifndef CONFIG_SLOB 4188 memcg->kmemcg_id = -1; 4189 #endif 4190 #ifdef CONFIG_CGROUP_WRITEBACK 4191 INIT_LIST_HEAD(&memcg->cgwb_list); 4192 #endif 4193 idr_replace(&mem_cgroup_idr, memcg, memcg->id.id); 4194 return memcg; 4195 fail: 4196 if (memcg->id.id > 0) 4197 idr_remove(&mem_cgroup_idr, memcg->id.id); 4198 mem_cgroup_free(memcg); 4199 return NULL; 4200 } 4201 4202 static struct cgroup_subsys_state * __ref 4203 mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 4204 { 4205 struct mem_cgroup *parent = mem_cgroup_from_css(parent_css); 4206 struct mem_cgroup *memcg; 4207 long error = -ENOMEM; 4208 4209 memcg = mem_cgroup_alloc(); 4210 if (!memcg) 4211 return ERR_PTR(error); 4212 4213 memcg->high = PAGE_COUNTER_MAX; 4214 memcg->soft_limit = PAGE_COUNTER_MAX; 4215 if (parent) { 4216 memcg->swappiness = mem_cgroup_swappiness(parent); 4217 memcg->oom_kill_disable = parent->oom_kill_disable; 4218 } 4219 if (parent && parent->use_hierarchy) { 4220 memcg->use_hierarchy = true; 4221 page_counter_init(&memcg->memory, &parent->memory); 4222 page_counter_init(&memcg->swap, &parent->swap); 4223 page_counter_init(&memcg->memsw, &parent->memsw); 4224 page_counter_init(&memcg->kmem, &parent->kmem); 4225 page_counter_init(&memcg->tcpmem, &parent->tcpmem); 4226 } else { 4227 page_counter_init(&memcg->memory, NULL); 4228 page_counter_init(&memcg->swap, NULL); 4229 page_counter_init(&memcg->memsw, NULL); 4230 page_counter_init(&memcg->kmem, NULL); 4231 page_counter_init(&memcg->tcpmem, NULL); 4232 /* 4233 * Deeper hierachy with use_hierarchy == false doesn't make 4234 * much sense so let cgroup subsystem know about this 4235 * unfortunate state in our controller. 4236 */ 4237 if (parent != root_mem_cgroup) 4238 memory_cgrp_subsys.broken_hierarchy = true; 4239 } 4240 4241 /* The following stuff does not apply to the root */ 4242 if (!parent) { 4243 root_mem_cgroup = memcg; 4244 return &memcg->css; 4245 } 4246 4247 error = memcg_online_kmem(memcg); 4248 if (error) 4249 goto fail; 4250 4251 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket) 4252 static_branch_inc(&memcg_sockets_enabled_key); 4253 4254 return &memcg->css; 4255 fail: 4256 mem_cgroup_free(memcg); 4257 return ERR_PTR(-ENOMEM); 4258 } 4259 4260 static int mem_cgroup_css_online(struct cgroup_subsys_state *css) 4261 { 4262 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4263 4264 /* Online state pins memcg ID, memcg ID pins CSS */ 4265 atomic_set(&memcg->id.ref, 1); 4266 css_get(css); 4267 return 0; 4268 } 4269 4270 static void mem_cgroup_css_offline(struct cgroup_subsys_state *css) 4271 { 4272 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4273 struct mem_cgroup_event *event, *tmp; 4274 4275 /* 4276 * Unregister events and notify userspace. 4277 * Notify userspace about cgroup removing only after rmdir of cgroup 4278 * directory to avoid race between userspace and kernelspace. 4279 */ 4280 spin_lock(&memcg->event_list_lock); 4281 list_for_each_entry_safe(event, tmp, &memcg->event_list, list) { 4282 list_del_init(&event->list); 4283 schedule_work(&event->remove); 4284 } 4285 spin_unlock(&memcg->event_list_lock); 4286 4287 memcg_offline_kmem(memcg); 4288 wb_memcg_offline(memcg); 4289 4290 mem_cgroup_id_put(memcg); 4291 } 4292 4293 static void mem_cgroup_css_released(struct cgroup_subsys_state *css) 4294 { 4295 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4296 4297 invalidate_reclaim_iterators(memcg); 4298 } 4299 4300 static void mem_cgroup_css_free(struct cgroup_subsys_state *css) 4301 { 4302 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4303 4304 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket) 4305 static_branch_dec(&memcg_sockets_enabled_key); 4306 4307 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active) 4308 static_branch_dec(&memcg_sockets_enabled_key); 4309 4310 vmpressure_cleanup(&memcg->vmpressure); 4311 cancel_work_sync(&memcg->high_work); 4312 mem_cgroup_remove_from_trees(memcg); 4313 memcg_free_kmem(memcg); 4314 mem_cgroup_free(memcg); 4315 } 4316 4317 /** 4318 * mem_cgroup_css_reset - reset the states of a mem_cgroup 4319 * @css: the target css 4320 * 4321 * Reset the states of the mem_cgroup associated with @css. This is 4322 * invoked when the userland requests disabling on the default hierarchy 4323 * but the memcg is pinned through dependency. The memcg should stop 4324 * applying policies and should revert to the vanilla state as it may be 4325 * made visible again. 4326 * 4327 * The current implementation only resets the essential configurations. 4328 * This needs to be expanded to cover all the visible parts. 4329 */ 4330 static void mem_cgroup_css_reset(struct cgroup_subsys_state *css) 4331 { 4332 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 4333 4334 page_counter_limit(&memcg->memory, PAGE_COUNTER_MAX); 4335 page_counter_limit(&memcg->swap, PAGE_COUNTER_MAX); 4336 page_counter_limit(&memcg->memsw, PAGE_COUNTER_MAX); 4337 page_counter_limit(&memcg->kmem, PAGE_COUNTER_MAX); 4338 page_counter_limit(&memcg->tcpmem, PAGE_COUNTER_MAX); 4339 memcg->low = 0; 4340 memcg->high = PAGE_COUNTER_MAX; 4341 memcg->soft_limit = PAGE_COUNTER_MAX; 4342 memcg_wb_domain_size_changed(memcg); 4343 } 4344 4345 #ifdef CONFIG_MMU 4346 /* Handlers for move charge at task migration. */ 4347 static int mem_cgroup_do_precharge(unsigned long count) 4348 { 4349 int ret; 4350 4351 /* Try a single bulk charge without reclaim first, kswapd may wake */ 4352 ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count); 4353 if (!ret) { 4354 mc.precharge += count; 4355 return ret; 4356 } 4357 4358 /* Try charges one by one with reclaim */ 4359 while (count--) { 4360 ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_NORETRY, 1); 4361 if (ret) 4362 return ret; 4363 mc.precharge++; 4364 cond_resched(); 4365 } 4366 return 0; 4367 } 4368 4369 union mc_target { 4370 struct page *page; 4371 swp_entry_t ent; 4372 }; 4373 4374 enum mc_target_type { 4375 MC_TARGET_NONE = 0, 4376 MC_TARGET_PAGE, 4377 MC_TARGET_SWAP, 4378 }; 4379 4380 static struct page *mc_handle_present_pte(struct vm_area_struct *vma, 4381 unsigned long addr, pte_t ptent) 4382 { 4383 struct page *page = vm_normal_page(vma, addr, ptent); 4384 4385 if (!page || !page_mapped(page)) 4386 return NULL; 4387 if (PageAnon(page)) { 4388 if (!(mc.flags & MOVE_ANON)) 4389 return NULL; 4390 } else { 4391 if (!(mc.flags & MOVE_FILE)) 4392 return NULL; 4393 } 4394 if (!get_page_unless_zero(page)) 4395 return NULL; 4396 4397 return page; 4398 } 4399 4400 #ifdef CONFIG_SWAP 4401 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma, 4402 pte_t ptent, swp_entry_t *entry) 4403 { 4404 struct page *page = NULL; 4405 swp_entry_t ent = pte_to_swp_entry(ptent); 4406 4407 if (!(mc.flags & MOVE_ANON) || non_swap_entry(ent)) 4408 return NULL; 4409 /* 4410 * Because lookup_swap_cache() updates some statistics counter, 4411 * we call find_get_page() with swapper_space directly. 4412 */ 4413 page = find_get_page(swap_address_space(ent), swp_offset(ent)); 4414 if (do_memsw_account()) 4415 entry->val = ent.val; 4416 4417 return page; 4418 } 4419 #else 4420 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma, 4421 pte_t ptent, swp_entry_t *entry) 4422 { 4423 return NULL; 4424 } 4425 #endif 4426 4427 static struct page *mc_handle_file_pte(struct vm_area_struct *vma, 4428 unsigned long addr, pte_t ptent, swp_entry_t *entry) 4429 { 4430 struct page *page = NULL; 4431 struct address_space *mapping; 4432 pgoff_t pgoff; 4433 4434 if (!vma->vm_file) /* anonymous vma */ 4435 return NULL; 4436 if (!(mc.flags & MOVE_FILE)) 4437 return NULL; 4438 4439 mapping = vma->vm_file->f_mapping; 4440 pgoff = linear_page_index(vma, addr); 4441 4442 /* page is moved even if it's not RSS of this task(page-faulted). */ 4443 #ifdef CONFIG_SWAP 4444 /* shmem/tmpfs may report page out on swap: account for that too. */ 4445 if (shmem_mapping(mapping)) { 4446 page = find_get_entry(mapping, pgoff); 4447 if (radix_tree_exceptional_entry(page)) { 4448 swp_entry_t swp = radix_to_swp_entry(page); 4449 if (do_memsw_account()) 4450 *entry = swp; 4451 page = find_get_page(swap_address_space(swp), 4452 swp_offset(swp)); 4453 } 4454 } else 4455 page = find_get_page(mapping, pgoff); 4456 #else 4457 page = find_get_page(mapping, pgoff); 4458 #endif 4459 return page; 4460 } 4461 4462 /** 4463 * mem_cgroup_move_account - move account of the page 4464 * @page: the page 4465 * @compound: charge the page as compound or small page 4466 * @from: mem_cgroup which the page is moved from. 4467 * @to: mem_cgroup which the page is moved to. @from != @to. 4468 * 4469 * The caller must make sure the page is not on LRU (isolate_page() is useful.) 4470 * 4471 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge" 4472 * from old cgroup. 4473 */ 4474 static int mem_cgroup_move_account(struct page *page, 4475 bool compound, 4476 struct mem_cgroup *from, 4477 struct mem_cgroup *to) 4478 { 4479 unsigned long flags; 4480 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1; 4481 int ret; 4482 bool anon; 4483 4484 VM_BUG_ON(from == to); 4485 VM_BUG_ON_PAGE(PageLRU(page), page); 4486 VM_BUG_ON(compound && !PageTransHuge(page)); 4487 4488 /* 4489 * Prevent mem_cgroup_migrate() from looking at 4490 * page->mem_cgroup of its source page while we change it. 4491 */ 4492 ret = -EBUSY; 4493 if (!trylock_page(page)) 4494 goto out; 4495 4496 ret = -EINVAL; 4497 if (page->mem_cgroup != from) 4498 goto out_unlock; 4499 4500 anon = PageAnon(page); 4501 4502 spin_lock_irqsave(&from->move_lock, flags); 4503 4504 if (!anon && page_mapped(page)) { 4505 __this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED], 4506 nr_pages); 4507 __this_cpu_add(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED], 4508 nr_pages); 4509 } 4510 4511 /* 4512 * move_lock grabbed above and caller set from->moving_account, so 4513 * mem_cgroup_update_page_stat() will serialize updates to PageDirty. 4514 * So mapping should be stable for dirty pages. 4515 */ 4516 if (!anon && PageDirty(page)) { 4517 struct address_space *mapping = page_mapping(page); 4518 4519 if (mapping_cap_account_dirty(mapping)) { 4520 __this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_DIRTY], 4521 nr_pages); 4522 __this_cpu_add(to->stat->count[MEM_CGROUP_STAT_DIRTY], 4523 nr_pages); 4524 } 4525 } 4526 4527 if (PageWriteback(page)) { 4528 __this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_WRITEBACK], 4529 nr_pages); 4530 __this_cpu_add(to->stat->count[MEM_CGROUP_STAT_WRITEBACK], 4531 nr_pages); 4532 } 4533 4534 /* 4535 * It is safe to change page->mem_cgroup here because the page 4536 * is referenced, charged, and isolated - we can't race with 4537 * uncharging, charging, migration, or LRU putback. 4538 */ 4539 4540 /* caller should have done css_get */ 4541 page->mem_cgroup = to; 4542 spin_unlock_irqrestore(&from->move_lock, flags); 4543 4544 ret = 0; 4545 4546 local_irq_disable(); 4547 mem_cgroup_charge_statistics(to, page, compound, nr_pages); 4548 memcg_check_events(to, page); 4549 mem_cgroup_charge_statistics(from, page, compound, -nr_pages); 4550 memcg_check_events(from, page); 4551 local_irq_enable(); 4552 out_unlock: 4553 unlock_page(page); 4554 out: 4555 return ret; 4556 } 4557 4558 /** 4559 * get_mctgt_type - get target type of moving charge 4560 * @vma: the vma the pte to be checked belongs 4561 * @addr: the address corresponding to the pte to be checked 4562 * @ptent: the pte to be checked 4563 * @target: the pointer the target page or swap ent will be stored(can be NULL) 4564 * 4565 * Returns 4566 * 0(MC_TARGET_NONE): if the pte is not a target for move charge. 4567 * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for 4568 * move charge. if @target is not NULL, the page is stored in target->page 4569 * with extra refcnt got(Callers should handle it). 4570 * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a 4571 * target for charge migration. if @target is not NULL, the entry is stored 4572 * in target->ent. 4573 * 4574 * Called with pte lock held. 4575 */ 4576 4577 static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma, 4578 unsigned long addr, pte_t ptent, union mc_target *target) 4579 { 4580 struct page *page = NULL; 4581 enum mc_target_type ret = MC_TARGET_NONE; 4582 swp_entry_t ent = { .val = 0 }; 4583 4584 if (pte_present(ptent)) 4585 page = mc_handle_present_pte(vma, addr, ptent); 4586 else if (is_swap_pte(ptent)) 4587 page = mc_handle_swap_pte(vma, ptent, &ent); 4588 else if (pte_none(ptent)) 4589 page = mc_handle_file_pte(vma, addr, ptent, &ent); 4590 4591 if (!page && !ent.val) 4592 return ret; 4593 if (page) { 4594 /* 4595 * Do only loose check w/o serialization. 4596 * mem_cgroup_move_account() checks the page is valid or 4597 * not under LRU exclusion. 4598 */ 4599 if (page->mem_cgroup == mc.from) { 4600 ret = MC_TARGET_PAGE; 4601 if (target) 4602 target->page = page; 4603 } 4604 if (!ret || !target) 4605 put_page(page); 4606 } 4607 /* There is a swap entry and a page doesn't exist or isn't charged */ 4608 if (ent.val && !ret && 4609 mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) { 4610 ret = MC_TARGET_SWAP; 4611 if (target) 4612 target->ent = ent; 4613 } 4614 return ret; 4615 } 4616 4617 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4618 /* 4619 * We don't consider swapping or file mapped pages because THP does not 4620 * support them for now. 4621 * Caller should make sure that pmd_trans_huge(pmd) is true. 4622 */ 4623 static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma, 4624 unsigned long addr, pmd_t pmd, union mc_target *target) 4625 { 4626 struct page *page = NULL; 4627 enum mc_target_type ret = MC_TARGET_NONE; 4628 4629 page = pmd_page(pmd); 4630 VM_BUG_ON_PAGE(!page || !PageHead(page), page); 4631 if (!(mc.flags & MOVE_ANON)) 4632 return ret; 4633 if (page->mem_cgroup == mc.from) { 4634 ret = MC_TARGET_PAGE; 4635 if (target) { 4636 get_page(page); 4637 target->page = page; 4638 } 4639 } 4640 return ret; 4641 } 4642 #else 4643 static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma, 4644 unsigned long addr, pmd_t pmd, union mc_target *target) 4645 { 4646 return MC_TARGET_NONE; 4647 } 4648 #endif 4649 4650 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd, 4651 unsigned long addr, unsigned long end, 4652 struct mm_walk *walk) 4653 { 4654 struct vm_area_struct *vma = walk->vma; 4655 pte_t *pte; 4656 spinlock_t *ptl; 4657 4658 ptl = pmd_trans_huge_lock(pmd, vma); 4659 if (ptl) { 4660 if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE) 4661 mc.precharge += HPAGE_PMD_NR; 4662 spin_unlock(ptl); 4663 return 0; 4664 } 4665 4666 if (pmd_trans_unstable(pmd)) 4667 return 0; 4668 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 4669 for (; addr != end; pte++, addr += PAGE_SIZE) 4670 if (get_mctgt_type(vma, addr, *pte, NULL)) 4671 mc.precharge++; /* increment precharge temporarily */ 4672 pte_unmap_unlock(pte - 1, ptl); 4673 cond_resched(); 4674 4675 return 0; 4676 } 4677 4678 static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm) 4679 { 4680 unsigned long precharge; 4681 4682 struct mm_walk mem_cgroup_count_precharge_walk = { 4683 .pmd_entry = mem_cgroup_count_precharge_pte_range, 4684 .mm = mm, 4685 }; 4686 down_read(&mm->mmap_sem); 4687 walk_page_range(0, mm->highest_vm_end, 4688 &mem_cgroup_count_precharge_walk); 4689 up_read(&mm->mmap_sem); 4690 4691 precharge = mc.precharge; 4692 mc.precharge = 0; 4693 4694 return precharge; 4695 } 4696 4697 static int mem_cgroup_precharge_mc(struct mm_struct *mm) 4698 { 4699 unsigned long precharge = mem_cgroup_count_precharge(mm); 4700 4701 VM_BUG_ON(mc.moving_task); 4702 mc.moving_task = current; 4703 return mem_cgroup_do_precharge(precharge); 4704 } 4705 4706 /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */ 4707 static void __mem_cgroup_clear_mc(void) 4708 { 4709 struct mem_cgroup *from = mc.from; 4710 struct mem_cgroup *to = mc.to; 4711 4712 /* we must uncharge all the leftover precharges from mc.to */ 4713 if (mc.precharge) { 4714 cancel_charge(mc.to, mc.precharge); 4715 mc.precharge = 0; 4716 } 4717 /* 4718 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so 4719 * we must uncharge here. 4720 */ 4721 if (mc.moved_charge) { 4722 cancel_charge(mc.from, mc.moved_charge); 4723 mc.moved_charge = 0; 4724 } 4725 /* we must fixup refcnts and charges */ 4726 if (mc.moved_swap) { 4727 /* uncharge swap account from the old cgroup */ 4728 if (!mem_cgroup_is_root(mc.from)) 4729 page_counter_uncharge(&mc.from->memsw, mc.moved_swap); 4730 4731 mem_cgroup_id_put_many(mc.from, mc.moved_swap); 4732 4733 /* 4734 * we charged both to->memory and to->memsw, so we 4735 * should uncharge to->memory. 4736 */ 4737 if (!mem_cgroup_is_root(mc.to)) 4738 page_counter_uncharge(&mc.to->memory, mc.moved_swap); 4739 4740 mem_cgroup_id_get_many(mc.to, mc.moved_swap); 4741 css_put_many(&mc.to->css, mc.moved_swap); 4742 4743 mc.moved_swap = 0; 4744 } 4745 memcg_oom_recover(from); 4746 memcg_oom_recover(to); 4747 wake_up_all(&mc.waitq); 4748 } 4749 4750 static void mem_cgroup_clear_mc(void) 4751 { 4752 struct mm_struct *mm = mc.mm; 4753 4754 /* 4755 * we must clear moving_task before waking up waiters at the end of 4756 * task migration. 4757 */ 4758 mc.moving_task = NULL; 4759 __mem_cgroup_clear_mc(); 4760 spin_lock(&mc.lock); 4761 mc.from = NULL; 4762 mc.to = NULL; 4763 mc.mm = NULL; 4764 spin_unlock(&mc.lock); 4765 4766 mmput(mm); 4767 } 4768 4769 static int mem_cgroup_can_attach(struct cgroup_taskset *tset) 4770 { 4771 struct cgroup_subsys_state *css; 4772 struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */ 4773 struct mem_cgroup *from; 4774 struct task_struct *leader, *p; 4775 struct mm_struct *mm; 4776 unsigned long move_flags; 4777 int ret = 0; 4778 4779 /* charge immigration isn't supported on the default hierarchy */ 4780 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 4781 return 0; 4782 4783 /* 4784 * Multi-process migrations only happen on the default hierarchy 4785 * where charge immigration is not used. Perform charge 4786 * immigration if @tset contains a leader and whine if there are 4787 * multiple. 4788 */ 4789 p = NULL; 4790 cgroup_taskset_for_each_leader(leader, css, tset) { 4791 WARN_ON_ONCE(p); 4792 p = leader; 4793 memcg = mem_cgroup_from_css(css); 4794 } 4795 if (!p) 4796 return 0; 4797 4798 /* 4799 * We are now commited to this value whatever it is. Changes in this 4800 * tunable will only affect upcoming migrations, not the current one. 4801 * So we need to save it, and keep it going. 4802 */ 4803 move_flags = READ_ONCE(memcg->move_charge_at_immigrate); 4804 if (!move_flags) 4805 return 0; 4806 4807 from = mem_cgroup_from_task(p); 4808 4809 VM_BUG_ON(from == memcg); 4810 4811 mm = get_task_mm(p); 4812 if (!mm) 4813 return 0; 4814 /* We move charges only when we move a owner of the mm */ 4815 if (mm->owner == p) { 4816 VM_BUG_ON(mc.from); 4817 VM_BUG_ON(mc.to); 4818 VM_BUG_ON(mc.precharge); 4819 VM_BUG_ON(mc.moved_charge); 4820 VM_BUG_ON(mc.moved_swap); 4821 4822 spin_lock(&mc.lock); 4823 mc.mm = mm; 4824 mc.from = from; 4825 mc.to = memcg; 4826 mc.flags = move_flags; 4827 spin_unlock(&mc.lock); 4828 /* We set mc.moving_task later */ 4829 4830 ret = mem_cgroup_precharge_mc(mm); 4831 if (ret) 4832 mem_cgroup_clear_mc(); 4833 } else { 4834 mmput(mm); 4835 } 4836 return ret; 4837 } 4838 4839 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset) 4840 { 4841 if (mc.to) 4842 mem_cgroup_clear_mc(); 4843 } 4844 4845 static int mem_cgroup_move_charge_pte_range(pmd_t *pmd, 4846 unsigned long addr, unsigned long end, 4847 struct mm_walk *walk) 4848 { 4849 int ret = 0; 4850 struct vm_area_struct *vma = walk->vma; 4851 pte_t *pte; 4852 spinlock_t *ptl; 4853 enum mc_target_type target_type; 4854 union mc_target target; 4855 struct page *page; 4856 4857 ptl = pmd_trans_huge_lock(pmd, vma); 4858 if (ptl) { 4859 if (mc.precharge < HPAGE_PMD_NR) { 4860 spin_unlock(ptl); 4861 return 0; 4862 } 4863 target_type = get_mctgt_type_thp(vma, addr, *pmd, &target); 4864 if (target_type == MC_TARGET_PAGE) { 4865 page = target.page; 4866 if (!isolate_lru_page(page)) { 4867 if (!mem_cgroup_move_account(page, true, 4868 mc.from, mc.to)) { 4869 mc.precharge -= HPAGE_PMD_NR; 4870 mc.moved_charge += HPAGE_PMD_NR; 4871 } 4872 putback_lru_page(page); 4873 } 4874 put_page(page); 4875 } 4876 spin_unlock(ptl); 4877 return 0; 4878 } 4879 4880 if (pmd_trans_unstable(pmd)) 4881 return 0; 4882 retry: 4883 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 4884 for (; addr != end; addr += PAGE_SIZE) { 4885 pte_t ptent = *(pte++); 4886 swp_entry_t ent; 4887 4888 if (!mc.precharge) 4889 break; 4890 4891 switch (get_mctgt_type(vma, addr, ptent, &target)) { 4892 case MC_TARGET_PAGE: 4893 page = target.page; 4894 /* 4895 * We can have a part of the split pmd here. Moving it 4896 * can be done but it would be too convoluted so simply 4897 * ignore such a partial THP and keep it in original 4898 * memcg. There should be somebody mapping the head. 4899 */ 4900 if (PageTransCompound(page)) 4901 goto put; 4902 if (isolate_lru_page(page)) 4903 goto put; 4904 if (!mem_cgroup_move_account(page, false, 4905 mc.from, mc.to)) { 4906 mc.precharge--; 4907 /* we uncharge from mc.from later. */ 4908 mc.moved_charge++; 4909 } 4910 putback_lru_page(page); 4911 put: /* get_mctgt_type() gets the page */ 4912 put_page(page); 4913 break; 4914 case MC_TARGET_SWAP: 4915 ent = target.ent; 4916 if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) { 4917 mc.precharge--; 4918 /* we fixup refcnts and charges later. */ 4919 mc.moved_swap++; 4920 } 4921 break; 4922 default: 4923 break; 4924 } 4925 } 4926 pte_unmap_unlock(pte - 1, ptl); 4927 cond_resched(); 4928 4929 if (addr != end) { 4930 /* 4931 * We have consumed all precharges we got in can_attach(). 4932 * We try charge one by one, but don't do any additional 4933 * charges to mc.to if we have failed in charge once in attach() 4934 * phase. 4935 */ 4936 ret = mem_cgroup_do_precharge(1); 4937 if (!ret) 4938 goto retry; 4939 } 4940 4941 return ret; 4942 } 4943 4944 static void mem_cgroup_move_charge(void) 4945 { 4946 struct mm_walk mem_cgroup_move_charge_walk = { 4947 .pmd_entry = mem_cgroup_move_charge_pte_range, 4948 .mm = mc.mm, 4949 }; 4950 4951 lru_add_drain_all(); 4952 /* 4953 * Signal lock_page_memcg() to take the memcg's move_lock 4954 * while we're moving its pages to another memcg. Then wait 4955 * for already started RCU-only updates to finish. 4956 */ 4957 atomic_inc(&mc.from->moving_account); 4958 synchronize_rcu(); 4959 retry: 4960 if (unlikely(!down_read_trylock(&mc.mm->mmap_sem))) { 4961 /* 4962 * Someone who are holding the mmap_sem might be waiting in 4963 * waitq. So we cancel all extra charges, wake up all waiters, 4964 * and retry. Because we cancel precharges, we might not be able 4965 * to move enough charges, but moving charge is a best-effort 4966 * feature anyway, so it wouldn't be a big problem. 4967 */ 4968 __mem_cgroup_clear_mc(); 4969 cond_resched(); 4970 goto retry; 4971 } 4972 /* 4973 * When we have consumed all precharges and failed in doing 4974 * additional charge, the page walk just aborts. 4975 */ 4976 walk_page_range(0, mc.mm->highest_vm_end, &mem_cgroup_move_charge_walk); 4977 4978 up_read(&mc.mm->mmap_sem); 4979 atomic_dec(&mc.from->moving_account); 4980 } 4981 4982 static void mem_cgroup_move_task(void) 4983 { 4984 if (mc.to) { 4985 mem_cgroup_move_charge(); 4986 mem_cgroup_clear_mc(); 4987 } 4988 } 4989 #else /* !CONFIG_MMU */ 4990 static int mem_cgroup_can_attach(struct cgroup_taskset *tset) 4991 { 4992 return 0; 4993 } 4994 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset) 4995 { 4996 } 4997 static void mem_cgroup_move_task(void) 4998 { 4999 } 5000 #endif 5001 5002 /* 5003 * Cgroup retains root cgroups across [un]mount cycles making it necessary 5004 * to verify whether we're attached to the default hierarchy on each mount 5005 * attempt. 5006 */ 5007 static void mem_cgroup_bind(struct cgroup_subsys_state *root_css) 5008 { 5009 /* 5010 * use_hierarchy is forced on the default hierarchy. cgroup core 5011 * guarantees that @root doesn't have any children, so turning it 5012 * on for the root memcg is enough. 5013 */ 5014 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 5015 root_mem_cgroup->use_hierarchy = true; 5016 else 5017 root_mem_cgroup->use_hierarchy = false; 5018 } 5019 5020 static u64 memory_current_read(struct cgroup_subsys_state *css, 5021 struct cftype *cft) 5022 { 5023 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 5024 5025 return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE; 5026 } 5027 5028 static int memory_low_show(struct seq_file *m, void *v) 5029 { 5030 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 5031 unsigned long low = READ_ONCE(memcg->low); 5032 5033 if (low == PAGE_COUNTER_MAX) 5034 seq_puts(m, "max\n"); 5035 else 5036 seq_printf(m, "%llu\n", (u64)low * PAGE_SIZE); 5037 5038 return 0; 5039 } 5040 5041 static ssize_t memory_low_write(struct kernfs_open_file *of, 5042 char *buf, size_t nbytes, loff_t off) 5043 { 5044 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 5045 unsigned long low; 5046 int err; 5047 5048 buf = strstrip(buf); 5049 err = page_counter_memparse(buf, "max", &low); 5050 if (err) 5051 return err; 5052 5053 memcg->low = low; 5054 5055 return nbytes; 5056 } 5057 5058 static int memory_high_show(struct seq_file *m, void *v) 5059 { 5060 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 5061 unsigned long high = READ_ONCE(memcg->high); 5062 5063 if (high == PAGE_COUNTER_MAX) 5064 seq_puts(m, "max\n"); 5065 else 5066 seq_printf(m, "%llu\n", (u64)high * PAGE_SIZE); 5067 5068 return 0; 5069 } 5070 5071 static ssize_t memory_high_write(struct kernfs_open_file *of, 5072 char *buf, size_t nbytes, loff_t off) 5073 { 5074 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 5075 unsigned long nr_pages; 5076 unsigned long high; 5077 int err; 5078 5079 buf = strstrip(buf); 5080 err = page_counter_memparse(buf, "max", &high); 5081 if (err) 5082 return err; 5083 5084 memcg->high = high; 5085 5086 nr_pages = page_counter_read(&memcg->memory); 5087 if (nr_pages > high) 5088 try_to_free_mem_cgroup_pages(memcg, nr_pages - high, 5089 GFP_KERNEL, true); 5090 5091 memcg_wb_domain_size_changed(memcg); 5092 return nbytes; 5093 } 5094 5095 static int memory_max_show(struct seq_file *m, void *v) 5096 { 5097 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 5098 unsigned long max = READ_ONCE(memcg->memory.limit); 5099 5100 if (max == PAGE_COUNTER_MAX) 5101 seq_puts(m, "max\n"); 5102 else 5103 seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE); 5104 5105 return 0; 5106 } 5107 5108 static ssize_t memory_max_write(struct kernfs_open_file *of, 5109 char *buf, size_t nbytes, loff_t off) 5110 { 5111 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 5112 unsigned int nr_reclaims = MEM_CGROUP_RECLAIM_RETRIES; 5113 bool drained = false; 5114 unsigned long max; 5115 int err; 5116 5117 buf = strstrip(buf); 5118 err = page_counter_memparse(buf, "max", &max); 5119 if (err) 5120 return err; 5121 5122 xchg(&memcg->memory.limit, max); 5123 5124 for (;;) { 5125 unsigned long nr_pages = page_counter_read(&memcg->memory); 5126 5127 if (nr_pages <= max) 5128 break; 5129 5130 if (signal_pending(current)) { 5131 err = -EINTR; 5132 break; 5133 } 5134 5135 if (!drained) { 5136 drain_all_stock(memcg); 5137 drained = true; 5138 continue; 5139 } 5140 5141 if (nr_reclaims) { 5142 if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max, 5143 GFP_KERNEL, true)) 5144 nr_reclaims--; 5145 continue; 5146 } 5147 5148 mem_cgroup_events(memcg, MEMCG_OOM, 1); 5149 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0)) 5150 break; 5151 } 5152 5153 memcg_wb_domain_size_changed(memcg); 5154 return nbytes; 5155 } 5156 5157 static int memory_events_show(struct seq_file *m, void *v) 5158 { 5159 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 5160 5161 seq_printf(m, "low %lu\n", mem_cgroup_read_events(memcg, MEMCG_LOW)); 5162 seq_printf(m, "high %lu\n", mem_cgroup_read_events(memcg, MEMCG_HIGH)); 5163 seq_printf(m, "max %lu\n", mem_cgroup_read_events(memcg, MEMCG_MAX)); 5164 seq_printf(m, "oom %lu\n", mem_cgroup_read_events(memcg, MEMCG_OOM)); 5165 5166 return 0; 5167 } 5168 5169 static int memory_stat_show(struct seq_file *m, void *v) 5170 { 5171 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 5172 unsigned long stat[MEMCG_NR_STAT]; 5173 unsigned long events[MEMCG_NR_EVENTS]; 5174 int i; 5175 5176 /* 5177 * Provide statistics on the state of the memory subsystem as 5178 * well as cumulative event counters that show past behavior. 5179 * 5180 * This list is ordered following a combination of these gradients: 5181 * 1) generic big picture -> specifics and details 5182 * 2) reflecting userspace activity -> reflecting kernel heuristics 5183 * 5184 * Current memory state: 5185 */ 5186 5187 tree_stat(memcg, stat); 5188 tree_events(memcg, events); 5189 5190 seq_printf(m, "anon %llu\n", 5191 (u64)stat[MEM_CGROUP_STAT_RSS] * PAGE_SIZE); 5192 seq_printf(m, "file %llu\n", 5193 (u64)stat[MEM_CGROUP_STAT_CACHE] * PAGE_SIZE); 5194 seq_printf(m, "kernel_stack %llu\n", 5195 (u64)stat[MEMCG_KERNEL_STACK_KB] * 1024); 5196 seq_printf(m, "slab %llu\n", 5197 (u64)(stat[MEMCG_SLAB_RECLAIMABLE] + 5198 stat[MEMCG_SLAB_UNRECLAIMABLE]) * PAGE_SIZE); 5199 seq_printf(m, "sock %llu\n", 5200 (u64)stat[MEMCG_SOCK] * PAGE_SIZE); 5201 5202 seq_printf(m, "file_mapped %llu\n", 5203 (u64)stat[MEM_CGROUP_STAT_FILE_MAPPED] * PAGE_SIZE); 5204 seq_printf(m, "file_dirty %llu\n", 5205 (u64)stat[MEM_CGROUP_STAT_DIRTY] * PAGE_SIZE); 5206 seq_printf(m, "file_writeback %llu\n", 5207 (u64)stat[MEM_CGROUP_STAT_WRITEBACK] * PAGE_SIZE); 5208 5209 for (i = 0; i < NR_LRU_LISTS; i++) { 5210 struct mem_cgroup *mi; 5211 unsigned long val = 0; 5212 5213 for_each_mem_cgroup_tree(mi, memcg) 5214 val += mem_cgroup_nr_lru_pages(mi, BIT(i)); 5215 seq_printf(m, "%s %llu\n", 5216 mem_cgroup_lru_names[i], (u64)val * PAGE_SIZE); 5217 } 5218 5219 seq_printf(m, "slab_reclaimable %llu\n", 5220 (u64)stat[MEMCG_SLAB_RECLAIMABLE] * PAGE_SIZE); 5221 seq_printf(m, "slab_unreclaimable %llu\n", 5222 (u64)stat[MEMCG_SLAB_UNRECLAIMABLE] * PAGE_SIZE); 5223 5224 /* Accumulated memory events */ 5225 5226 seq_printf(m, "pgfault %lu\n", 5227 events[MEM_CGROUP_EVENTS_PGFAULT]); 5228 seq_printf(m, "pgmajfault %lu\n", 5229 events[MEM_CGROUP_EVENTS_PGMAJFAULT]); 5230 5231 return 0; 5232 } 5233 5234 static struct cftype memory_files[] = { 5235 { 5236 .name = "current", 5237 .flags = CFTYPE_NOT_ON_ROOT, 5238 .read_u64 = memory_current_read, 5239 }, 5240 { 5241 .name = "low", 5242 .flags = CFTYPE_NOT_ON_ROOT, 5243 .seq_show = memory_low_show, 5244 .write = memory_low_write, 5245 }, 5246 { 5247 .name = "high", 5248 .flags = CFTYPE_NOT_ON_ROOT, 5249 .seq_show = memory_high_show, 5250 .write = memory_high_write, 5251 }, 5252 { 5253 .name = "max", 5254 .flags = CFTYPE_NOT_ON_ROOT, 5255 .seq_show = memory_max_show, 5256 .write = memory_max_write, 5257 }, 5258 { 5259 .name = "events", 5260 .flags = CFTYPE_NOT_ON_ROOT, 5261 .file_offset = offsetof(struct mem_cgroup, events_file), 5262 .seq_show = memory_events_show, 5263 }, 5264 { 5265 .name = "stat", 5266 .flags = CFTYPE_NOT_ON_ROOT, 5267 .seq_show = memory_stat_show, 5268 }, 5269 { } /* terminate */ 5270 }; 5271 5272 struct cgroup_subsys memory_cgrp_subsys = { 5273 .css_alloc = mem_cgroup_css_alloc, 5274 .css_online = mem_cgroup_css_online, 5275 .css_offline = mem_cgroup_css_offline, 5276 .css_released = mem_cgroup_css_released, 5277 .css_free = mem_cgroup_css_free, 5278 .css_reset = mem_cgroup_css_reset, 5279 .can_attach = mem_cgroup_can_attach, 5280 .cancel_attach = mem_cgroup_cancel_attach, 5281 .post_attach = mem_cgroup_move_task, 5282 .bind = mem_cgroup_bind, 5283 .dfl_cftypes = memory_files, 5284 .legacy_cftypes = mem_cgroup_legacy_files, 5285 .early_init = 0, 5286 }; 5287 5288 /** 5289 * mem_cgroup_low - check if memory consumption is below the normal range 5290 * @root: the highest ancestor to consider 5291 * @memcg: the memory cgroup to check 5292 * 5293 * Returns %true if memory consumption of @memcg, and that of all 5294 * configurable ancestors up to @root, is below the normal range. 5295 */ 5296 bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg) 5297 { 5298 if (mem_cgroup_disabled()) 5299 return false; 5300 5301 /* 5302 * The toplevel group doesn't have a configurable range, so 5303 * it's never low when looked at directly, and it is not 5304 * considered an ancestor when assessing the hierarchy. 5305 */ 5306 5307 if (memcg == root_mem_cgroup) 5308 return false; 5309 5310 if (page_counter_read(&memcg->memory) >= memcg->low) 5311 return false; 5312 5313 while (memcg != root) { 5314 memcg = parent_mem_cgroup(memcg); 5315 5316 if (memcg == root_mem_cgroup) 5317 break; 5318 5319 if (page_counter_read(&memcg->memory) >= memcg->low) 5320 return false; 5321 } 5322 return true; 5323 } 5324 5325 /** 5326 * mem_cgroup_try_charge - try charging a page 5327 * @page: page to charge 5328 * @mm: mm context of the victim 5329 * @gfp_mask: reclaim mode 5330 * @memcgp: charged memcg return 5331 * @compound: charge the page as compound or small page 5332 * 5333 * Try to charge @page to the memcg that @mm belongs to, reclaiming 5334 * pages according to @gfp_mask if necessary. 5335 * 5336 * Returns 0 on success, with *@memcgp pointing to the charged memcg. 5337 * Otherwise, an error code is returned. 5338 * 5339 * After page->mapping has been set up, the caller must finalize the 5340 * charge with mem_cgroup_commit_charge(). Or abort the transaction 5341 * with mem_cgroup_cancel_charge() in case page instantiation fails. 5342 */ 5343 int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm, 5344 gfp_t gfp_mask, struct mem_cgroup **memcgp, 5345 bool compound) 5346 { 5347 struct mem_cgroup *memcg = NULL; 5348 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1; 5349 int ret = 0; 5350 5351 if (mem_cgroup_disabled()) 5352 goto out; 5353 5354 if (PageSwapCache(page)) { 5355 /* 5356 * Every swap fault against a single page tries to charge the 5357 * page, bail as early as possible. shmem_unuse() encounters 5358 * already charged pages, too. The USED bit is protected by 5359 * the page lock, which serializes swap cache removal, which 5360 * in turn serializes uncharging. 5361 */ 5362 VM_BUG_ON_PAGE(!PageLocked(page), page); 5363 if (page->mem_cgroup) 5364 goto out; 5365 5366 if (do_swap_account) { 5367 swp_entry_t ent = { .val = page_private(page), }; 5368 unsigned short id = lookup_swap_cgroup_id(ent); 5369 5370 rcu_read_lock(); 5371 memcg = mem_cgroup_from_id(id); 5372 if (memcg && !css_tryget_online(&memcg->css)) 5373 memcg = NULL; 5374 rcu_read_unlock(); 5375 } 5376 } 5377 5378 if (!memcg) 5379 memcg = get_mem_cgroup_from_mm(mm); 5380 5381 ret = try_charge(memcg, gfp_mask, nr_pages); 5382 5383 css_put(&memcg->css); 5384 out: 5385 *memcgp = memcg; 5386 return ret; 5387 } 5388 5389 /** 5390 * mem_cgroup_commit_charge - commit a page charge 5391 * @page: page to charge 5392 * @memcg: memcg to charge the page to 5393 * @lrucare: page might be on LRU already 5394 * @compound: charge the page as compound or small page 5395 * 5396 * Finalize a charge transaction started by mem_cgroup_try_charge(), 5397 * after page->mapping has been set up. This must happen atomically 5398 * as part of the page instantiation, i.e. under the page table lock 5399 * for anonymous pages, under the page lock for page and swap cache. 5400 * 5401 * In addition, the page must not be on the LRU during the commit, to 5402 * prevent racing with task migration. If it might be, use @lrucare. 5403 * 5404 * Use mem_cgroup_cancel_charge() to cancel the transaction instead. 5405 */ 5406 void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg, 5407 bool lrucare, bool compound) 5408 { 5409 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1; 5410 5411 VM_BUG_ON_PAGE(!page->mapping, page); 5412 VM_BUG_ON_PAGE(PageLRU(page) && !lrucare, page); 5413 5414 if (mem_cgroup_disabled()) 5415 return; 5416 /* 5417 * Swap faults will attempt to charge the same page multiple 5418 * times. But reuse_swap_page() might have removed the page 5419 * from swapcache already, so we can't check PageSwapCache(). 5420 */ 5421 if (!memcg) 5422 return; 5423 5424 commit_charge(page, memcg, lrucare); 5425 5426 local_irq_disable(); 5427 mem_cgroup_charge_statistics(memcg, page, compound, nr_pages); 5428 memcg_check_events(memcg, page); 5429 local_irq_enable(); 5430 5431 if (do_memsw_account() && PageSwapCache(page)) { 5432 swp_entry_t entry = { .val = page_private(page) }; 5433 /* 5434 * The swap entry might not get freed for a long time, 5435 * let's not wait for it. The page already received a 5436 * memory+swap charge, drop the swap entry duplicate. 5437 */ 5438 mem_cgroup_uncharge_swap(entry); 5439 } 5440 } 5441 5442 /** 5443 * mem_cgroup_cancel_charge - cancel a page charge 5444 * @page: page to charge 5445 * @memcg: memcg to charge the page to 5446 * @compound: charge the page as compound or small page 5447 * 5448 * Cancel a charge transaction started by mem_cgroup_try_charge(). 5449 */ 5450 void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg, 5451 bool compound) 5452 { 5453 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1; 5454 5455 if (mem_cgroup_disabled()) 5456 return; 5457 /* 5458 * Swap faults will attempt to charge the same page multiple 5459 * times. But reuse_swap_page() might have removed the page 5460 * from swapcache already, so we can't check PageSwapCache(). 5461 */ 5462 if (!memcg) 5463 return; 5464 5465 cancel_charge(memcg, nr_pages); 5466 } 5467 5468 static void uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout, 5469 unsigned long nr_anon, unsigned long nr_file, 5470 unsigned long nr_huge, unsigned long nr_kmem, 5471 struct page *dummy_page) 5472 { 5473 unsigned long nr_pages = nr_anon + nr_file + nr_kmem; 5474 unsigned long flags; 5475 5476 if (!mem_cgroup_is_root(memcg)) { 5477 page_counter_uncharge(&memcg->memory, nr_pages); 5478 if (do_memsw_account()) 5479 page_counter_uncharge(&memcg->memsw, nr_pages); 5480 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && nr_kmem) 5481 page_counter_uncharge(&memcg->kmem, nr_kmem); 5482 memcg_oom_recover(memcg); 5483 } 5484 5485 local_irq_save(flags); 5486 __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS], nr_anon); 5487 __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_CACHE], nr_file); 5488 __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], nr_huge); 5489 __this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT], pgpgout); 5490 __this_cpu_add(memcg->stat->nr_page_events, nr_pages); 5491 memcg_check_events(memcg, dummy_page); 5492 local_irq_restore(flags); 5493 5494 if (!mem_cgroup_is_root(memcg)) 5495 css_put_many(&memcg->css, nr_pages); 5496 } 5497 5498 static void uncharge_list(struct list_head *page_list) 5499 { 5500 struct mem_cgroup *memcg = NULL; 5501 unsigned long nr_anon = 0; 5502 unsigned long nr_file = 0; 5503 unsigned long nr_huge = 0; 5504 unsigned long nr_kmem = 0; 5505 unsigned long pgpgout = 0; 5506 struct list_head *next; 5507 struct page *page; 5508 5509 /* 5510 * Note that the list can be a single page->lru; hence the 5511 * do-while loop instead of a simple list_for_each_entry(). 5512 */ 5513 next = page_list->next; 5514 do { 5515 page = list_entry(next, struct page, lru); 5516 next = page->lru.next; 5517 5518 VM_BUG_ON_PAGE(PageLRU(page), page); 5519 VM_BUG_ON_PAGE(page_count(page), page); 5520 5521 if (!page->mem_cgroup) 5522 continue; 5523 5524 /* 5525 * Nobody should be changing or seriously looking at 5526 * page->mem_cgroup at this point, we have fully 5527 * exclusive access to the page. 5528 */ 5529 5530 if (memcg != page->mem_cgroup) { 5531 if (memcg) { 5532 uncharge_batch(memcg, pgpgout, nr_anon, nr_file, 5533 nr_huge, nr_kmem, page); 5534 pgpgout = nr_anon = nr_file = 5535 nr_huge = nr_kmem = 0; 5536 } 5537 memcg = page->mem_cgroup; 5538 } 5539 5540 if (!PageKmemcg(page)) { 5541 unsigned int nr_pages = 1; 5542 5543 if (PageTransHuge(page)) { 5544 nr_pages <<= compound_order(page); 5545 nr_huge += nr_pages; 5546 } 5547 if (PageAnon(page)) 5548 nr_anon += nr_pages; 5549 else 5550 nr_file += nr_pages; 5551 pgpgout++; 5552 } else { 5553 nr_kmem += 1 << compound_order(page); 5554 __ClearPageKmemcg(page); 5555 } 5556 5557 page->mem_cgroup = NULL; 5558 } while (next != page_list); 5559 5560 if (memcg) 5561 uncharge_batch(memcg, pgpgout, nr_anon, nr_file, 5562 nr_huge, nr_kmem, page); 5563 } 5564 5565 /** 5566 * mem_cgroup_uncharge - uncharge a page 5567 * @page: page to uncharge 5568 * 5569 * Uncharge a page previously charged with mem_cgroup_try_charge() and 5570 * mem_cgroup_commit_charge(). 5571 */ 5572 void mem_cgroup_uncharge(struct page *page) 5573 { 5574 if (mem_cgroup_disabled()) 5575 return; 5576 5577 /* Don't touch page->lru of any random page, pre-check: */ 5578 if (!page->mem_cgroup) 5579 return; 5580 5581 INIT_LIST_HEAD(&page->lru); 5582 uncharge_list(&page->lru); 5583 } 5584 5585 /** 5586 * mem_cgroup_uncharge_list - uncharge a list of page 5587 * @page_list: list of pages to uncharge 5588 * 5589 * Uncharge a list of pages previously charged with 5590 * mem_cgroup_try_charge() and mem_cgroup_commit_charge(). 5591 */ 5592 void mem_cgroup_uncharge_list(struct list_head *page_list) 5593 { 5594 if (mem_cgroup_disabled()) 5595 return; 5596 5597 if (!list_empty(page_list)) 5598 uncharge_list(page_list); 5599 } 5600 5601 /** 5602 * mem_cgroup_migrate - charge a page's replacement 5603 * @oldpage: currently circulating page 5604 * @newpage: replacement page 5605 * 5606 * Charge @newpage as a replacement page for @oldpage. @oldpage will 5607 * be uncharged upon free. 5608 * 5609 * Both pages must be locked, @newpage->mapping must be set up. 5610 */ 5611 void mem_cgroup_migrate(struct page *oldpage, struct page *newpage) 5612 { 5613 struct mem_cgroup *memcg; 5614 unsigned int nr_pages; 5615 bool compound; 5616 unsigned long flags; 5617 5618 VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage); 5619 VM_BUG_ON_PAGE(!PageLocked(newpage), newpage); 5620 VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage); 5621 VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage), 5622 newpage); 5623 5624 if (mem_cgroup_disabled()) 5625 return; 5626 5627 /* Page cache replacement: new page already charged? */ 5628 if (newpage->mem_cgroup) 5629 return; 5630 5631 /* Swapcache readahead pages can get replaced before being charged */ 5632 memcg = oldpage->mem_cgroup; 5633 if (!memcg) 5634 return; 5635 5636 /* Force-charge the new page. The old one will be freed soon */ 5637 compound = PageTransHuge(newpage); 5638 nr_pages = compound ? hpage_nr_pages(newpage) : 1; 5639 5640 page_counter_charge(&memcg->memory, nr_pages); 5641 if (do_memsw_account()) 5642 page_counter_charge(&memcg->memsw, nr_pages); 5643 css_get_many(&memcg->css, nr_pages); 5644 5645 commit_charge(newpage, memcg, false); 5646 5647 local_irq_save(flags); 5648 mem_cgroup_charge_statistics(memcg, newpage, compound, nr_pages); 5649 memcg_check_events(memcg, newpage); 5650 local_irq_restore(flags); 5651 } 5652 5653 DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key); 5654 EXPORT_SYMBOL(memcg_sockets_enabled_key); 5655 5656 void mem_cgroup_sk_alloc(struct sock *sk) 5657 { 5658 struct mem_cgroup *memcg; 5659 5660 if (!mem_cgroup_sockets_enabled) 5661 return; 5662 5663 /* 5664 * Socket cloning can throw us here with sk_memcg already 5665 * filled. It won't however, necessarily happen from 5666 * process context. So the test for root memcg given 5667 * the current task's memcg won't help us in this case. 5668 * 5669 * Respecting the original socket's memcg is a better 5670 * decision in this case. 5671 */ 5672 if (sk->sk_memcg) { 5673 BUG_ON(mem_cgroup_is_root(sk->sk_memcg)); 5674 css_get(&sk->sk_memcg->css); 5675 return; 5676 } 5677 5678 rcu_read_lock(); 5679 memcg = mem_cgroup_from_task(current); 5680 if (memcg == root_mem_cgroup) 5681 goto out; 5682 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active) 5683 goto out; 5684 if (css_tryget_online(&memcg->css)) 5685 sk->sk_memcg = memcg; 5686 out: 5687 rcu_read_unlock(); 5688 } 5689 5690 void mem_cgroup_sk_free(struct sock *sk) 5691 { 5692 if (sk->sk_memcg) 5693 css_put(&sk->sk_memcg->css); 5694 } 5695 5696 /** 5697 * mem_cgroup_charge_skmem - charge socket memory 5698 * @memcg: memcg to charge 5699 * @nr_pages: number of pages to charge 5700 * 5701 * Charges @nr_pages to @memcg. Returns %true if the charge fit within 5702 * @memcg's configured limit, %false if the charge had to be forced. 5703 */ 5704 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages) 5705 { 5706 gfp_t gfp_mask = GFP_KERNEL; 5707 5708 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) { 5709 struct page_counter *fail; 5710 5711 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) { 5712 memcg->tcpmem_pressure = 0; 5713 return true; 5714 } 5715 page_counter_charge(&memcg->tcpmem, nr_pages); 5716 memcg->tcpmem_pressure = 1; 5717 return false; 5718 } 5719 5720 /* Don't block in the packet receive path */ 5721 if (in_softirq()) 5722 gfp_mask = GFP_NOWAIT; 5723 5724 this_cpu_add(memcg->stat->count[MEMCG_SOCK], nr_pages); 5725 5726 if (try_charge(memcg, gfp_mask, nr_pages) == 0) 5727 return true; 5728 5729 try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages); 5730 return false; 5731 } 5732 5733 /** 5734 * mem_cgroup_uncharge_skmem - uncharge socket memory 5735 * @memcg - memcg to uncharge 5736 * @nr_pages - number of pages to uncharge 5737 */ 5738 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages) 5739 { 5740 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) { 5741 page_counter_uncharge(&memcg->tcpmem, nr_pages); 5742 return; 5743 } 5744 5745 this_cpu_sub(memcg->stat->count[MEMCG_SOCK], nr_pages); 5746 5747 page_counter_uncharge(&memcg->memory, nr_pages); 5748 css_put_many(&memcg->css, nr_pages); 5749 } 5750 5751 static int __init cgroup_memory(char *s) 5752 { 5753 char *token; 5754 5755 while ((token = strsep(&s, ",")) != NULL) { 5756 if (!*token) 5757 continue; 5758 if (!strcmp(token, "nosocket")) 5759 cgroup_memory_nosocket = true; 5760 if (!strcmp(token, "nokmem")) 5761 cgroup_memory_nokmem = true; 5762 } 5763 return 0; 5764 } 5765 __setup("cgroup.memory=", cgroup_memory); 5766 5767 /* 5768 * subsys_initcall() for memory controller. 5769 * 5770 * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this 5771 * context because of lock dependencies (cgroup_lock -> cpu hotplug) but 5772 * basically everything that doesn't depend on a specific mem_cgroup structure 5773 * should be initialized from here. 5774 */ 5775 static int __init mem_cgroup_init(void) 5776 { 5777 int cpu, node; 5778 5779 #ifndef CONFIG_SLOB 5780 /* 5781 * Kmem cache creation is mostly done with the slab_mutex held, 5782 * so use a special workqueue to avoid stalling all worker 5783 * threads in case lots of cgroups are created simultaneously. 5784 */ 5785 memcg_kmem_cache_create_wq = 5786 alloc_ordered_workqueue("memcg_kmem_cache_create", 0); 5787 BUG_ON(!memcg_kmem_cache_create_wq); 5788 #endif 5789 5790 cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL, 5791 memcg_hotplug_cpu_dead); 5792 5793 for_each_possible_cpu(cpu) 5794 INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work, 5795 drain_local_stock); 5796 5797 for_each_node(node) { 5798 struct mem_cgroup_tree_per_node *rtpn; 5799 5800 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, 5801 node_online(node) ? node : NUMA_NO_NODE); 5802 5803 rtpn->rb_root = RB_ROOT; 5804 spin_lock_init(&rtpn->lock); 5805 soft_limit_tree.rb_tree_per_node[node] = rtpn; 5806 } 5807 5808 return 0; 5809 } 5810 subsys_initcall(mem_cgroup_init); 5811 5812 #ifdef CONFIG_MEMCG_SWAP 5813 static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg) 5814 { 5815 while (!atomic_inc_not_zero(&memcg->id.ref)) { 5816 /* 5817 * The root cgroup cannot be destroyed, so it's refcount must 5818 * always be >= 1. 5819 */ 5820 if (WARN_ON_ONCE(memcg == root_mem_cgroup)) { 5821 VM_BUG_ON(1); 5822 break; 5823 } 5824 memcg = parent_mem_cgroup(memcg); 5825 if (!memcg) 5826 memcg = root_mem_cgroup; 5827 } 5828 return memcg; 5829 } 5830 5831 /** 5832 * mem_cgroup_swapout - transfer a memsw charge to swap 5833 * @page: page whose memsw charge to transfer 5834 * @entry: swap entry to move the charge to 5835 * 5836 * Transfer the memsw charge of @page to @entry. 5837 */ 5838 void mem_cgroup_swapout(struct page *page, swp_entry_t entry) 5839 { 5840 struct mem_cgroup *memcg, *swap_memcg; 5841 unsigned short oldid; 5842 5843 VM_BUG_ON_PAGE(PageLRU(page), page); 5844 VM_BUG_ON_PAGE(page_count(page), page); 5845 5846 if (!do_memsw_account()) 5847 return; 5848 5849 memcg = page->mem_cgroup; 5850 5851 /* Readahead page, never charged */ 5852 if (!memcg) 5853 return; 5854 5855 /* 5856 * In case the memcg owning these pages has been offlined and doesn't 5857 * have an ID allocated to it anymore, charge the closest online 5858 * ancestor for the swap instead and transfer the memory+swap charge. 5859 */ 5860 swap_memcg = mem_cgroup_id_get_online(memcg); 5861 oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg)); 5862 VM_BUG_ON_PAGE(oldid, page); 5863 mem_cgroup_swap_statistics(swap_memcg, true); 5864 5865 page->mem_cgroup = NULL; 5866 5867 if (!mem_cgroup_is_root(memcg)) 5868 page_counter_uncharge(&memcg->memory, 1); 5869 5870 if (memcg != swap_memcg) { 5871 if (!mem_cgroup_is_root(swap_memcg)) 5872 page_counter_charge(&swap_memcg->memsw, 1); 5873 page_counter_uncharge(&memcg->memsw, 1); 5874 } 5875 5876 /* 5877 * Interrupts should be disabled here because the caller holds the 5878 * mapping->tree_lock lock which is taken with interrupts-off. It is 5879 * important here to have the interrupts disabled because it is the 5880 * only synchronisation we have for udpating the per-CPU variables. 5881 */ 5882 VM_BUG_ON(!irqs_disabled()); 5883 mem_cgroup_charge_statistics(memcg, page, false, -1); 5884 memcg_check_events(memcg, page); 5885 5886 if (!mem_cgroup_is_root(memcg)) 5887 css_put(&memcg->css); 5888 } 5889 5890 /* 5891 * mem_cgroup_try_charge_swap - try charging a swap entry 5892 * @page: page being added to swap 5893 * @entry: swap entry to charge 5894 * 5895 * Try to charge @entry to the memcg that @page belongs to. 5896 * 5897 * Returns 0 on success, -ENOMEM on failure. 5898 */ 5899 int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry) 5900 { 5901 struct mem_cgroup *memcg; 5902 struct page_counter *counter; 5903 unsigned short oldid; 5904 5905 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) || !do_swap_account) 5906 return 0; 5907 5908 memcg = page->mem_cgroup; 5909 5910 /* Readahead page, never charged */ 5911 if (!memcg) 5912 return 0; 5913 5914 memcg = mem_cgroup_id_get_online(memcg); 5915 5916 if (!mem_cgroup_is_root(memcg) && 5917 !page_counter_try_charge(&memcg->swap, 1, &counter)) { 5918 mem_cgroup_id_put(memcg); 5919 return -ENOMEM; 5920 } 5921 5922 oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg)); 5923 VM_BUG_ON_PAGE(oldid, page); 5924 mem_cgroup_swap_statistics(memcg, true); 5925 5926 return 0; 5927 } 5928 5929 /** 5930 * mem_cgroup_uncharge_swap - uncharge a swap entry 5931 * @entry: swap entry to uncharge 5932 * 5933 * Drop the swap charge associated with @entry. 5934 */ 5935 void mem_cgroup_uncharge_swap(swp_entry_t entry) 5936 { 5937 struct mem_cgroup *memcg; 5938 unsigned short id; 5939 5940 if (!do_swap_account) 5941 return; 5942 5943 id = swap_cgroup_record(entry, 0); 5944 rcu_read_lock(); 5945 memcg = mem_cgroup_from_id(id); 5946 if (memcg) { 5947 if (!mem_cgroup_is_root(memcg)) { 5948 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 5949 page_counter_uncharge(&memcg->swap, 1); 5950 else 5951 page_counter_uncharge(&memcg->memsw, 1); 5952 } 5953 mem_cgroup_swap_statistics(memcg, false); 5954 mem_cgroup_id_put(memcg); 5955 } 5956 rcu_read_unlock(); 5957 } 5958 5959 long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg) 5960 { 5961 long nr_swap_pages = get_nr_swap_pages(); 5962 5963 if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys)) 5964 return nr_swap_pages; 5965 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) 5966 nr_swap_pages = min_t(long, nr_swap_pages, 5967 READ_ONCE(memcg->swap.limit) - 5968 page_counter_read(&memcg->swap)); 5969 return nr_swap_pages; 5970 } 5971 5972 bool mem_cgroup_swap_full(struct page *page) 5973 { 5974 struct mem_cgroup *memcg; 5975 5976 VM_BUG_ON_PAGE(!PageLocked(page), page); 5977 5978 if (vm_swap_full()) 5979 return true; 5980 if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys)) 5981 return false; 5982 5983 memcg = page->mem_cgroup; 5984 if (!memcg) 5985 return false; 5986 5987 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) 5988 if (page_counter_read(&memcg->swap) * 2 >= memcg->swap.limit) 5989 return true; 5990 5991 return false; 5992 } 5993 5994 /* for remember boot option*/ 5995 #ifdef CONFIG_MEMCG_SWAP_ENABLED 5996 static int really_do_swap_account __initdata = 1; 5997 #else 5998 static int really_do_swap_account __initdata; 5999 #endif 6000 6001 static int __init enable_swap_account(char *s) 6002 { 6003 if (!strcmp(s, "1")) 6004 really_do_swap_account = 1; 6005 else if (!strcmp(s, "0")) 6006 really_do_swap_account = 0; 6007 return 1; 6008 } 6009 __setup("swapaccount=", enable_swap_account); 6010 6011 static u64 swap_current_read(struct cgroup_subsys_state *css, 6012 struct cftype *cft) 6013 { 6014 struct mem_cgroup *memcg = mem_cgroup_from_css(css); 6015 6016 return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE; 6017 } 6018 6019 static int swap_max_show(struct seq_file *m, void *v) 6020 { 6021 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 6022 unsigned long max = READ_ONCE(memcg->swap.limit); 6023 6024 if (max == PAGE_COUNTER_MAX) 6025 seq_puts(m, "max\n"); 6026 else 6027 seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE); 6028 6029 return 0; 6030 } 6031 6032 static ssize_t swap_max_write(struct kernfs_open_file *of, 6033 char *buf, size_t nbytes, loff_t off) 6034 { 6035 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); 6036 unsigned long max; 6037 int err; 6038 6039 buf = strstrip(buf); 6040 err = page_counter_memparse(buf, "max", &max); 6041 if (err) 6042 return err; 6043 6044 mutex_lock(&memcg_limit_mutex); 6045 err = page_counter_limit(&memcg->swap, max); 6046 mutex_unlock(&memcg_limit_mutex); 6047 if (err) 6048 return err; 6049 6050 return nbytes; 6051 } 6052 6053 static struct cftype swap_files[] = { 6054 { 6055 .name = "swap.current", 6056 .flags = CFTYPE_NOT_ON_ROOT, 6057 .read_u64 = swap_current_read, 6058 }, 6059 { 6060 .name = "swap.max", 6061 .flags = CFTYPE_NOT_ON_ROOT, 6062 .seq_show = swap_max_show, 6063 .write = swap_max_write, 6064 }, 6065 { } /* terminate */ 6066 }; 6067 6068 static struct cftype memsw_cgroup_files[] = { 6069 { 6070 .name = "memsw.usage_in_bytes", 6071 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE), 6072 .read_u64 = mem_cgroup_read_u64, 6073 }, 6074 { 6075 .name = "memsw.max_usage_in_bytes", 6076 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE), 6077 .write = mem_cgroup_reset, 6078 .read_u64 = mem_cgroup_read_u64, 6079 }, 6080 { 6081 .name = "memsw.limit_in_bytes", 6082 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT), 6083 .write = mem_cgroup_write, 6084 .read_u64 = mem_cgroup_read_u64, 6085 }, 6086 { 6087 .name = "memsw.failcnt", 6088 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT), 6089 .write = mem_cgroup_reset, 6090 .read_u64 = mem_cgroup_read_u64, 6091 }, 6092 { }, /* terminate */ 6093 }; 6094 6095 static int __init mem_cgroup_swap_init(void) 6096 { 6097 if (!mem_cgroup_disabled() && really_do_swap_account) { 6098 do_swap_account = 1; 6099 WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, 6100 swap_files)); 6101 WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys, 6102 memsw_cgroup_files)); 6103 } 6104 return 0; 6105 } 6106 subsys_initcall(mem_cgroup_swap_init); 6107 6108 #endif /* CONFIG_MEMCG_SWAP */ 6109